カテゴリー: Development

def __tokenize_partial(self, text, wakati, baseform_unk, dotfile):
    if self.wakati and not wakati:
        raise WakatiModeOnlyException

    chunk_size = min(len(text), Tokenizer.MAX_CHUNK_SIZE)
    lattice = Lattice(chunk_size, self.sys_dic)
    pos = 0
    while not self.__should_split(text, pos):
        encoded_partial_text = text[pos:pos + min(50, chunk_size - pos)].encode('utf-8')
        # user dictionary
        if self.user_dic:
            entries = self.user_dic.lookup(encoded_partial_text)
            for e in entries:
                lattice.add(SurfaceNode(e, NodeType.USER_DICT))
            matched = len(entries) > 0

        # system dictionary
        entries = self.sys_dic.lookup(encoded_partial_text, self.matcher)
        for e in entries:
            lattice.add(SurfaceNode(e, NodeType.SYS_DICT))
        matched = len(entries) > 0

        # unknown
        cates = self.sys_dic.get_char_categories(text[pos])
        if cates:
            for cate in cates:
                if matched and not self.sys_dic.unknown_invoked_always(cate):
                    continue
                # unknown word length
                length = self.sys_dic.unknown_length(cate) \
                    if not self.sys_dic.unknown_grouping(cate) else self.max_unknown_length
                assert length >= 0
                # buffer for unknown word
                buf = text[pos]
                for p in range(pos + 1, min(chunk_size, pos + length + 1)):
                    _cates = self.sys_dic.get_char_categories(text[p])
                    if cate in _cates or any(cate in _compat_cates for _compat_cates in _cates.values()):
                        buf += text[p]
                    else:
                        break
                unknown_entries = self.sys_dic.unknowns.get(cate)
                assert unknown_entries
                for entry in unknown_entries:
                    left_id, right_id, cost, part_of_speech = entry
                    base_form = buf if baseform_unk else '*'
                    dummy_dict_entry = (buf, left_id, right_id, cost, part_of_speech, '*', '*', base_form, '*', '*')
                    lattice.add(Node(dummy_dict_entry, NodeType.UNKNOWN))

        pos += lattice.forward()
    lattice.end()
    min_cost_path = lattice.backward()
    assert isinstance(min_cost_path[0], BOS)
    assert isinstance(min_cost_path[-1], EOS)
    if wakati:
        tokens = [node.surface for node in min_cost_path[1:-1]]
    else:
        tokens = []
        for node in min_cost_path[1:-1]:
            if type(node) == SurfaceNode and node.node_type == NodeType.SYS_DICT:
                tokens.append(Token(node, self.sys_dic.lookup_extra(node.num)))
            elif type(node) == SurfaceNode and node.node_type == NodeType.USER_DICT:
                tokens.append(Token(node, self.user_dic.lookup_extra(node.num)))
            else:
                tokens.append(Token(node))
    if dotfile:
        lattice.generate_dotfile(filename=dotfile)
    return (tokens, pos)

# start drawing grey image
for word, freq in frequencies:
    if freq == 0:
        continue
    # select the font size
    rs = self.relative_scaling
    if rs != 0:
        font_size = int(round((rs * (freq / float(last_freq))
                                + (1 - rs)) * font_size))
    if random_state.random() < self.prefer_horizontal:
        orientation = None
    else:
        orientation = Image.ROTATE_90
    tried_other_orientation = False
    while True:
        # try to find a position
        font = ImageFont.truetype(self.font_path, font_size)
        # transpose font optionally
        transposed_font = ImageFont.TransposedFont(
            font, orientation=orientation)
        # get size of resulting text
        box_size = draw.textbbox((0, 0), word, font=transposed_font, anchor="lt")
        # find possible places using integral image:
        result = occupancy.sample_position(box_size[3] + self.margin,
                                            box_size[2] + self.margin,
                                            random_state)
・・・省略・・・

ext_modules=[Extension("wordcloud.query_integral_image",
                       ["wordcloud/query_integral_image.c"])],

# cython: language_level=3
# cython: boundscheck=False
# cython: wraparound=False
import array
import numpy as np


def query_integral_image(unsigned int[:,:] integral_image, int size_x, int
                         size_y, random_state):
    cdef int x = integral_image.shape[0]
    cdef int y = integral_image.shape[1]
    cdef int area, i, j
    cdef int hits = 0

    # count how many possible locations
    for i in xrange(x - size_x):
        for j in xrange(y - size_y):
            area = integral_image[i, j] + integral_image[i + size_x, j + size_y]
            area -= integral_image[i + size_x, j] + integral_image[i, j + size_y]
            if not area:
                hits += 1
    if not hits:
        # no room left
        return None
    # pick a location at random
    cdef int goal = random_state.randint(0, hits)
    hits = 0
    for i in xrange(x - size_x):
        for j in xrange(y - size_y):
            area = integral_image[i, j] + integral_image[i + size_x, j + size_y]
            area -= integral_image[i + size_x, j] + integral_image[i, j + size_y]
            if not area:
                hits += 1
                if hits == goal:
                    return i, j

area = integral_image[i, j] + integral_image[i + size_x, j + size_y]
area -= integral_image[i + size_x, j] + integral_image[i, j + size_y]