human
Complete this python program:
def compute_coocs(lines, vocab):
assistant
#Here is the Python program you requested.

    """Compute the co-occurrence statistics from the text.
    This generates a temporary file for each shard that contains the intermediate
    counts from the shard: these counts must be subsequently sorted and collated.
    """
    word_to_id = {tok: idx for (idx, tok) in enumerate(vocab)}
    lines.seek(0, os.SEEK_END)
    nbytes = lines.tell()
    lines.seek(0, os.SEEK_SET)
    num_shards = (len(vocab) / FLAGS.shard_size)
    shardfiles = {}
    for row in range(num_shards):
        for col in range(num_shards):
            filename = os.path.join(FLAGS.output_dir, ('shard-%03d-%03d.tmp' % (row, col)))
            shardfiles[(row, col)] = open(filename, 'w+')
    def flush_coocs():
        for ((row_id, col_id), cnt) in coocs.iteritems():
            row_shard = (row_id % num_shards)
            row_off = (row_id / num_shards)
            col_shard = (col_id % num_shards)
            col_off = (col_id / num_shards)
            shardfiles[(row_shard, col_shard)].write(shard_cooc_fmt.pack(row_off, col_off, cnt))
            shardfiles[(col_shard, row_shard)].write(shard_cooc_fmt.pack(col_off, row_off, cnt))
    coocs = {}
    sums = ([0.0] * len(vocab))
    for (lineno, line) in enumerate(lines, start=1):
        wids = filter((lambda wid: (wid is not None)), (word_to_id.get(w) for w in words(line)))
        for pos in xrange(len(wids)):
            lid = wids[pos]
            window_extent = min((FLAGS.window_size + 1), (len(wids) - pos))
            for off in xrange(1, window_extent):
                rid = wids[(pos + off)]
                pair = (min(lid, rid), max(lid, rid))
                count = (1.0 / off)
                sums[lid] += count
                sums[rid] += count
                coocs.setdefault(pair, 0.0)
                coocs[pair] += count
            sums[lid] += 1.0
            pair = (lid, lid)
            coocs.setdefault(pair, 0.0)
            coocs[pair] += 0.5
        if ((lineno % 10000) == 0):
            pos = lines.tell()
            sys.stdout.write(('\rComputing    co-occurrences:    %0.1f%%    (%d/%d)...' % (((100.0 * pos) / nbytes), pos, nbytes)))
            sys.stdout.flush()
            if (len(coocs) > FLAGS.bufsz):
                flush_coocs()
                coocs = {}
    flush_coocs()
...