text:
#!/usr/bin/env python3

import sys
import os
import shutil
import cv2
from ament_index_python import get_package_share_directory
import yaml
import numpy as np
from transforms3d.euler import euler2mat, mat2euler
from subprocess import check_output, run
from copy import deepcopy
import xacro
from urdf_parser_py.urdf import URDF

marine_ppt = get_package_share_directory('marine_presenter')

def add_icon(img, x, y, w):
    im = cv2.imread(img)
    H,W = im.shape[:2]
    X = int(W*x/100.)
    Y = int(H*y/100.)
    w /= 100.
    icon = cv2.imread(marine_ppt + '/objects/video.png')
    scale = w*W/icon.shape[1]
    icon = cv2.resize(icon, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)
    im[Y:Y+icon.shape[0], X-icon.shape[1]:X] = icon
    cv2.imwrite(img, im)

def add_borders(img, w = 20):
    im = cv2.imread(img)
    H,W = im.shape[:2]
    w = int(W*(1+w/100.))
    im2 = np.full((H,W+w,3), 255, dtype=np.uint8)
    im2[:,w//2:W+w//2] = im
    cv2.imwrite(img, im2)

# get initial pdf
filename = sys.argv[1]
if filename[-4:] != '.pdf':
    filename = (filename+'.pdf').replace('..','.')

if not os.path.exists(filename):
    print(filename + ' does not exist, exiting')
    sys.exit(0)

# check config file
if os.path.exists(filename.replace('.pdf', '.yaml')):
    config = yaml.safe_load(open(filename.replace('.pdf', '.yaml')))
else:
    config = {}

video_x = 99.5
video_y = 0.5
video_w = 3
if 'video' in config:
    video_x = config['video']['x']
    video_y = config['video']['y']
    video_w = config['video']['w']

if 'scale' not in config:
    config['scale'] = 3

scale = config['scale']

ext = 'png'
cam_pose = [1, 0, scale, 0, 0, 0]    # wrt current slide

def dict_replace(s, d):
    for key in d:
        s = s.replace(key, d[key])
    return s

def read_pose(pose, base_pose = (0,0,0,0,0,0)):
    if type(pose) == dict:
        if 'pose' in pose:
            return read_pose(pose['pose'], base_pose)
        return base_pose

    if len(pose) >= 3:
        pose[2] *= scale
    if len(pose) == 6:
        return pose
    # complete with base_pose
    return pose + base_pose[len(pose)-6:]

def Homogeneous(pose):
    R = euler2mat(pose[3], pose[4], pose[5], 'rxyz')
    t = np.array(pose[:3]).reshape(3,1)
    M = np.vstack((np.hstack((R, t)), [0,0,0,1]))
    for i in range(3):
        for j in range(3):
            for v in [-1,0,1]:
                if abs(M[i,j]-v) < 1e-3:
                    M[i,j] = v
    return np.matrix(M)

def rec_insert(d, keys, val):
    key = keys.pop(0)
    if not len(keys):
        d[key] = val
        return
    if key not in d:
        d[key] = {}
    rec_insert(d[key], keys, val)

def resolve(xacro_file):
    pkg, path = xacro_file[10:].split('/',1)
    pkg = get_package_share_directory(pkg)

    xacro_file = os.path.join(pkg,path)

    if os.path.exists(xacro_file):
        return xacro_file

    # path is a raw file, look for it
    for root, dirs, files in os.walk(pkg, topdown=False):
        if path in files:
            return os.path.join(root, path)

    return xacro_file

basename = os.path.abspath(filename[:-4])
mesh_path = basename + '/mesh'
img_path = mesh_path + '/img'
config_out = basename + '/config.yaml'
generate = '-f' in sys.argv

def slide_img(slide):
    return '{}/{}.{}'.format(img_path, slide, ext)

for path in (basename, mesh_path, img_path):
    if not os.path.exists(path):
        os.mkdir(path)
        generate = True

# get PDF pages
pages = check_output(['pdfinfo',filename]).decode('utf-8').splitlines()
pages = [line for line in pages if line.startswith('Pages:')][0]
pages = int(pages.replace('Pages:','').replace(' ',''))

# change config to page-numbered, check videos

titles = {}
for slide in range(1,pages+1):
    title = check_output(['pdftotext','-f', str(slide), '-l', str(slide),'-layout', '{}'.format(filename),'-']).decode('utf-8')
    title = title[:title.find('\n')]
    if [ord(c) for c in title[-2:]] == [8722,49]:
        title = title[:-5]
    if title not in titles:
        titles[title] = [slide]
    else:
        titles[title].append(slide)

for title, slides in titles.items():

    base_slide = slides[0]

    if title in config:
        config[base_slide] = config.pop(title)

    if base_slide in config:

        slide_videos = [key for key in config[base_slide] if key.startswith('video')]
        for video in slide_videos:
            video_path = config[base_slide].pop(video)
            print(f'  found video: {video_path}')
            if not video_path.startswith('/'):
                video_path = os.path.abspath(os.path.dirname(basename) + '/' + video_path)
            if video == 'video':
                for slide in slides:
                    rec_insert(config, [slide, 'video'], video_path)
            else:
                offset = int(video[5:])-1
                rec_insert(config, [base_slide+offset, 'video'], video_path)

videos = sorted([key for key in config if type(key) == int and 'video' in config[key]])
print('Found {} slides'.format(pages))

pdf_change = True
video_change = True
scale_change = True
if os.path.exists(config_out):
    prev = yaml.safe_load(open(config_out))
    if 'pdf_time' in prev:
        pdf_change = prev['pdf_time'] != int(os.stat(filename).st_mtime)
    video_change = videos != sorted([key for key in prev if type(key) == int and 'video' in prev[key]])
    scale_change = scale != prev['scale']
config['pdf_time'] = int(os.stat(filename).st_mtime)

if pdf_change or video_change or generate:
    print('{} change: generating base images...'.format(pdf_change and 'PDF' or 'Videos'))
    os.system('pdftoppm {} -{} -scale-to 2048 {}/'.format(filename, ext if ext == 'png' else 'jpeg', img_path))

    for img in os.listdir(img_path):
        os.rename('{}/{}'.format(img_path, img), '{}/{}'.format(img_path, img.strip('-0')))

    # add video tag / borders
    for idx in videos:
        add_icon(slide_img(idx), video_x, video_y, video_w)

    if 'fit' in config:
        for slide in config['fit']:
            add_borders(slide_img(slide))

config['slides'] = pages
if 'cam_pose' not in config:
    config['cam_pose'] = cam_pose

# build meshs
print('Generating osg meshes...')

base_mesh = open(marine_ppt + '/templates/slide.osg').read()

pose = read_pose(config)
if 'pose' in config:
    config.pop('pose')

for slide in range(1, pages+1):
    im = cv2.imread(slide_img(slide))
    H,W = im.shape[:2]
    ratio = round(float(W)/H,2)

    dest = '{}/{}.osg'.format(mesh_path,str(slide))
    if scale_change or not os.path.exists(dest):
        open(dest, 'w').write(base_mesh.replace('<image>', 'img/{}.{}'.format(slide, ext)).replace('<x>', str(ratio*scale)).replace('<z>', str(scale)))

    # add pose information
    if slide not in config:
        config[slide] = {}
    if 'pose' in config[slide]:
        pose = read_pose(config[slide], pose)
    config[slide]['pose'] = pose

# objects are found by the presenter node, just adapt pose / scale and find files

if 'objects' in config:
    print('Resolving object paths...')
    for name in config['objects']:
        data = config['objects'][name]
        xacro_file = data['file'] if 'file' in data else name

        if xacro_file.startswith('package'):
            xacro_file = resolve(xacro_file)
            data['file'] = xacro_file

        if '/' not in xacro_file:
            marine_files = os.listdir(marine_ppt + '/objects/urdf')
            # find relative to marine_ppt
            l = len(xacro_file)
            found = False

            while l and not found:
                if xacro_file[:l] + '.xacro' in marine_files:
                    print('  Found object "{}", available in marine_presenter as {}.xacro'.format(name, xacro_file[:l]))
                    xacro_file = '{}/objects/urdf/{}.xacro'.format(marine_ppt, xacro_file[:l])
                    found = True
                l -= 1

            if not found:
                # look for local objects
                l = len(xacro_file)
                while l and not found:
                    for root, dirs, files in os.walk(os.path.dirname(basename), topdown=False):
                        for ext in ('.xacro','.urdf'):
                            candidate = xacro_file[:l] + ext
                            if candidate in files:
                                print('  Found object "{}", found locally as {}'.format(name, candidate))
                                xacro_file = os.path.join(root, candidate)
                                found = True
                                break
                        if found:
                            break
                    l -= 1

            if not found:
                print('  Cannot find object "{}" in marine_presenter or local path'.format(name))

            data['file'] = xacro_file

        else:
            # just check the file exists and trust the user
            if os.path.exists(xacro_file):
                print('  Found object "{}" at {}'.format(name, xacro_file))
            else:
                print('  Cannot find object "{}", should be at {}'.format(name, xacro_file))

        # find pose and scale
        pose = [0,0,0,0,0,0]
        if 'center' in data:
            pose = read_pose(data['center'],pose)
        if 'slide' in data:
            if type(data['slide']) == str:
                data['slide'] = titles[data['slide']][0]
            # pose is actually relative to slide
            M = Homogeneous(config[data['slide']]['pose']) * Homogeneous(pose)
            for i in range(3):
                pose[i] = float(M[i,3])
            pose[3:] = mat2euler(M)
        data['center'] = [v for v in pose]
        for v in ('rx','ry'):
            if v in data:
                data[v] *= scale

yaml.safe_dump(config, open(config_out, 'w'), default_flow_style=False)

print('Generating launch file...')

manual = open(marine_ppt + '/templates/manual.yaml').read()
with open(basename + '/manual.yaml', 'w') as f:
    f.write(manual.replace('<slides>', str(pages)))

launch = open(marine_ppt + '/templates/presentation_launch.py').read()

objects = {}
fixed_tfs = {}

if 'objects' in config:
    for name, params in config['objects'].items():
        objects[name] = params['file']
        # check if static
        if 'rx' not in params:
            description = xacro.process(params['file'], mappings={'prefix': name})
            link = URDF.from_xml_string(description).get_root()
            fixed_tfs[link] = [params['center'][i] for i in [0,1,2,5,4,3]]

with open(basename + '/presentation_launch.py', 'w') as f:
    f.write(launch.replace('<objects>', str(objects)).replace('<fixed_tfs>', str(fixed_tfs)))

print('\r done.')

if '-p' in sys.argv:
    os.system('ros2 launch ' + basename + '/presentation_launch.py')


text:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

dados = pd.read_csv('imoveis_sicar_uf_20180531.csv', delimiter = ';')

plt.figure(figsize = (20, 5))
plt.bar(dados.nlargest(10, 'imoveis_car')['sigla'], dados.nlargest(10, 'imoveis_car')['imoveis_car'], color = plt.cm.Set2(np.arange(10)))
plt.title('Os 10 estados com o maior número de imóveis registrados', fontsize = 16, fontweight = 'bold')
plt.show();

plt.figure(figsize = (20, 5))
plt.bar(dados.nsmallest(10, 'imoveis_car')['sigla'], dados.nsmallest(10, 'imoveis_car')['imoveis_car'], color = plt.cm.Set1(np.arange(10)))
plt.title('Os 10 estados com o menor número de imóveis registrados', fontsize = 16, fontweight = 'bold')
plt.show();

plt.figure(figsize = (20, 5))
plt.bar(dados.nlargest(10, 'area_car')['sigla'], dados.nlargest(10, 'area_car')['area_car'], color = plt.cm.Set2(np.arange(10)))
plt.title('Os 10 estados com a maior área dos imóveis registrados', fontsize = 16, fontweight = 'bold')
plt.show();

plt.figure(figsize = (20, 5))
plt.bar(dados.nsmallest(10, 'area_car')['sigla'], dados.nsmallest(10, 'area_car')['area_car'], color = plt.cm.Set1(np.arange(10)))
plt.title('Os 10 estados com a menor área dos imóveis registrados', fontsize = 16, fontweight = 'bold')
plt.show();

#Criação de uma nova coluna com médio de área de cada imóvel cadastrado
for index, row in dados.iterrows():
    dados.at[index, 'valor_unitario'] = row['area_car'] / row['imoveis_car']

plt.figure(figsize = (20, 5))
plt.bar(dados.nlargest(10, 'valor_unitario')['sigla'], dados.nlargest(10, 'valor_unitario')['valor_unitario'], color = plt.cm.Set2(np.arange(10)))
plt.title('Os 10 estados com o maior média de área por imóvel registrado', fontsize = 16, fontweight = 'bold')
plt.show();

plt.figure(figsize = (20, 5))
plt.bar(dados.nsmallest(10, 'valor_unitario')['sigla'], dados.nsmallest(10, 'valor_unitario')['valor_unitario'], color = plt.cm.Set1(np.arange(10)))
plt.title('Os 10 estados com a menor média de área por imóvel registrado', fontsize = 16, fontweight = 'bold')
plt.show();

ext: py

max_stars_repo_path: panorama geral.py

max_issues_repo_path: panorama geral.py

max_forks_repo_path: panorama geral.py

reason: import numpy


text:
import numpy as np
from ringity.classes.diagram import PersistenceDiagram

def read_pdiagram(fname, **kwargs):
    """
    Wrapper for numpy.genfromtxt.
    """
    return PersistenceDiagram(np.genfromtxt(fname, **kwargs))

def write_pdiagram(dgm, fname, **kwargs):
    """
    Wrapper for numpy.savetxt.
    """
    array = np.array(dgm)
    np.savetxt(fname, array, **kwargs)


text:
# Lab 9 XOR
import tensorflow as tf
import numpy as np

tf.set_random_seed(777)  # for reproducibility

x_data = [[0, 0],
          [0, 1],
          [1, 0],
          [1, 1]]
y_data = [[0],
          [1],
          [1],
          [0]]
x_data = np.array(x_data, dtype=np.float32)
y_data = np.array(y_data, dtype=np.float32)

X = tf.placeholder(tf.float32, [None, 2])
Y = tf.placeholder(tf.float32, [None, 1])

W = tf.Variable(tf.random_normal([2, 1]), name='weight')
b = tf.Variable(tf.random_normal([1]), name='bias')

# Hypothesis using sigmoid: tf.div(1., 1. + tf.exp(tf.matmul(X, W)))
hypothesis = tf.sigmoid(tf.matmul(X, W) + b)

# cost/loss function
cost = -tf.reduce_mean(Y * tf.log(hypothesis) + (1 - Y) *
                       tf.log(1 - hypothesis))

train = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(cost)

# Accuracy computation
# True if hypothesis>0.5 else False
predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, Y), dtype=tf.float32))

# Launch graph
with tf.Session() as sess:
    # Initialize TensorFlow variables
    sess.run(tf.global_variables_initializer())

    for step in range(10001):
        sess.run(train, feed_dict={X: x_data, Y: y_data})
        if step % 100 == 0:
            print(step, sess.run(cost, feed_dict={
                  X: x_data, Y: y_data}), sess.run(W))

    # Accuracy report
    h, c, a = sess.run([hypothesis, predicted, accuracy],
                       feed_dict={X: x_data, Y: y_data})
    print("\nHypothesis: ", h, "\nCorrect: ", c, "\nAccuracy: ", a)

max_stars_repo_name: fomuon/tensorflow-exams

max_issues_repo_name: fomuon/tensorflow-exams

max_forks_repo_name: fomuon/tensorflow-exams


text:
from __future__ import division

import torch
import model_caller
import numpy as np
import h5py
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from mpl_toolkits.mplot3d import Axes3D
import viz
import time
import os
import copy
import data_utils
import forward_kinematics

def main():
    parent, offset, rotInd, expmapInd = forward_kinematics._some_variables()

    # directions, 1, 1, 180, 8, 10, 10 flips out

    angles_to_wiggle = range(56, 70, 1)

    # === Plot and animate ===
    fig = plt.figure()
    ax = plt.gca(projection='3d')
    ob = viz.Ax3DPose(ax)

    # Plot the prediction
    for i in angles_to_wiggle:
        print("Wiggling",i,"...")
        time = 0
        pose = np.zeros((99)) * -.15
        for j in range(99):
            #if j%3 == 0 and j > 5:
            #    pose[j] = 1.57
            pass
            #pose[j] += j*.003
        pose[77] = -1.57
        pose[53] = 1.57
        pose[80] = -1.57
        while time < 25:
            pose[i] = 1.5*np.abs(np.sin(time))
            xyz = forward_kinematics.fkl(pose, parent, offset, rotInd, expmapInd)
            time += 0.25

            ob.update(xyz, rcolor="#f06090", lcolor="#6090b0")
            plt.show(block=False)
            fig.canvas.draw()

            plt.pause(0.02)

if __name__ == '__main__':
    main()


text:
import cv2
import os, logging, time, json
import requests, base64
from flask import Flask, jsonify, request, Response
import numpy as np

# for HTTP/1.1 support
from werkzeug.serving import WSGIRequestHandler

app = Flask(__name__)

logging.basicConfig(format='%(asctime)s  %(levelname)-10s %(message)s', datefmt="%Y-%m-%d-%H-%M-%S",
    level=logging.INFO)

def main():
    pass

def grab_image_from_stream():

    repeat = 3
    wait = 3
    frame = None

    for _ in range(repeat):

        try:
            video_capture = cv2.VideoCapture(args.camera)
            video_capture.set(cv2.CAP_PROP_BUFFERSIZE, 1)

            frame = video_capture.read()[1]
            break
        except:
            # try to re-capture the stream
            logging.info("Could not capture video. Recapturing and retrying...")
            time.sleep(wait)

    if frame is None:
        logging.info("Failed to capture frame, sending blank image")
        frame = np.zeros((300, 300, 3))

    return frame

@app.route('/image/700')
def video_image():

    frame = grab_image_from_stream()
    _, jpeg = cv2.imencode('.jpg', frame)
    response = Response(jpeg.tobytes(), headers={"content-length": len(jpeg)}, mimetype="image/jpeg")
    return response

@app.route('/image/800')
def video_image_and_inference():
    frame = grab_image_from_stream()
    frame = cv2.resize(frame, (300, 300))

    _, jpeg = cv2.imencode('.jpg', frame)
    resp_img = jpeg.tobytes()

    scoring_url = "http://grocerymodel:5001/score"
    json_img = json.dumps({"img": frame.tolist()})
    input_data = json_img
    headers = {'Content-Type':'application/json'}
    resp = requests.post(scoring_url, input_data, headers=headers)
    logging.info(f'received response: {resp.status_code}')
    resp_json = json.loads(resp.content)

    resp_json["img"] = str(base64.b64encode(resp_img), "utf-8")

    return jsonify(resp_json)

def start_app():
    # set protocol to 1.1 so we keep the connection open
    WSGIRequestHandler.protocol_version = "HTTP/1.1"

    if args.fast:
        logging.info("Running the `fast` version")

        app.run(host="0.0.0.0", port=args.port)
    else:
        logging.info(f"Staring regular inventory cam. Port: {args.port}")
        app.run(debug=False)

if __name__ == "__main__":
    from cmdline import cmd_args

    args = cmd_args.parse_camera_args()

    if not args.fast:
        app.config['SERVER_NAME'] = f'inventorycam:{args.port}'

    if args.debug:
        logging.info("Please attach a debugger to port 5678")

        import ptvsd
        ptvsd.enable_attach(('0.0.0.0', 5681))
        ptvsd.wait_for_attach()
        ptvsd.break_into_debugger()

    start_app()


text:
from pathlib import Path

import click
import numpy as np
import pandas as pd

def _save_datasets(train, test, outdir: Path, flag):
    """Save data sets into nice directory structure and write SUCCESS flag."""
    # csv paths and flag path
    out_train = outdir / "train.csv/"
    out_test = outdir / "test.csv/"
    flag = outdir / flag

    # save as csv and create flag file
    train.to_csv(str(out_train), index=False)
    test.to_csv(str(out_test), index=False)

    flag.touch()

# @click.command()
# @click.option('--in-csv')
# @click.option('--out-dir')
# @click.option('--flag')
def make_datasets(in_csv, out_dir, flag):
    out_dir = Path(out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    # create pandas dataframe
    ddf = pd.read_csv(in_csv, index_col="Unnamed: 0")

    # trigger computation
    n_samples = len(ddf)

    # TODO: implement proper dataset creation here
    # http://docs.dask.org/en/latest/dataframe-api.html

    # split dataset into train test feel free to adjust test percentage
    idx = np.arange(n_samples)

    # separate first 1000 rows as test set
    test_idx = idx[: n_samples // 10]
    test = ddf.loc[test_idx]

    # separate last 9000 rows as training set
    train_idx = idx[n_samples // 10 :]
    train = ddf.loc[train_idx]

    _save_datasets(train, test, out_dir, flag)

if __name__ == "__main__":
    make_datasets(
        in_csv="~/dags/pipeline/dataset/raw/wine_dataset.csv",
        out_dir="~/dags/pipeline/dataset/interim/",
        flag=".SUCCESS_MakeDatasets",
    )

max_stars_repo_name: reginold/airflow-handson

max_issues_repo_name: reginold/airflow-handson

max_forks_repo_name: reginold/airflow-handson


text:
from random import (
    choice,
    randrange,
    sample,
)
from numpy import arange
from datetime import datetime, timedelta
import pytz

utc = pytz.UTC

def create_mock_profile():
    currency = "BDT"
    account_balance = 0

    credit_card_balance = {}
    transaction_history = {"spend": {}, "deposit": {}}

    credit_card_db = [
        "brac bank",
        "platinum",
        "signature",
        "gold",
    ]
    deposit_db = [
        "employer",
        "interest",
    ]
    recipient_db = [
        "md ismail",
        "tarun kumar",
        "nur alam",
        "maksudur rahman",
        "amit ranjan",
        "anup paul",
        "emrul kayes",
        "ashraful alam",
    ]
    vendor_db = [
        "daraz",
        "chaldal",
        "pikaboo",
        "swapno"
    ]

    start_date = utc.localize(datetime(2020, 1, 1))
    end_date = utc.localize(datetime.now())
    number_of_days = (end_date - start_date).days

    for vendor in vendor_db:
        rand_spend_amounts = sample(
            [round(amount, 2) for amount in list(arange(500, 5000, 1.5))],
            number_of_days // 2,
        )

        rand_dates = [
            (
                start_date + timedelta(days=randrange(number_of_days))
            ).isoformat()
            for x in range(0, len(rand_spend_amounts))
        ]

        transaction_history["spend"][vendor] = [
            {"amount": amount, "date": date}
            for amount, date in zip(rand_spend_amounts, rand_dates)
        ]
        account_balance -= sum(rand_spend_amounts)

    for deposit in deposit_db:
        if deposit == "interest":
            rand_deposit_amounts = sample(
                [round(amount, 2) for amount in list(arange(500, 2000, 1.50))],
                number_of_days // 30,
            )
        else:
            rand_deposit_amounts = sample(
                [
                    round(amount, 2)
                    for amount in list(arange(100000, 200000, 1.50))
                ],
                number_of_days // 14,
            )

        rand_dates = [
            (
                start_date + timedelta(days=randrange(number_of_days))
            ).isoformat()
            for x in range(0, len(rand_deposit_amounts))
        ]

        transaction_history["deposit"][deposit] = [
            {"amount": amount, "date": date}
            for amount, date in zip(rand_deposit_amounts, rand_dates)
        ]
        account_balance += sum(rand_deposit_amounts) - sum(rand_spend_amounts)

    for credit_card in credit_card_db:
        credit_card_balance[credit_card] = {
            "minimum balance": 2000,
            "current balance": choice(
                [round(amount, 2) for amount in list(arange(2000, 50000, 1.5))]
            ),
        }

    mock_profile = {
        "account_balance": f"{account_balance:.2f}",
        "currency": currency,
        "transaction_history": transaction_history,
        "credit_card_balance": credit_card_balance,
        "known_recipients": [recipient.title() for recipient in recipient_db],
        "vendor_list": vendor_db,
    }
    return mock_profile

reason: from numpy


text:
# -*- coding: utf-8 -*-
"""

Core: The core functionality for Vincent to map to Vega grammar

"""
from __future__ import (print_function, division)
import json
from string import Template
from pkg_resources import resource_string

try:
    import pandas as pd
except ImportError:
    pd = None

try:
    import numpy as np
except ImportError:
    np = None

from ._compat import str_types

def initialize_notebook():
    """Initialize the IPython notebook display elements"""
    try:
        from IPython.core.display import display, HTML
    except ImportError:
        print("IPython Notebook could not be loaded.")

    # Thanks to @jakevdp:
    # https://github.com/jakevdp/mpld3/blob/master/mpld3/_display.py#L85
    load_lib = """
                function vct_load_lib(url, callback){
                      if(
                typeof d3 !== 'undefined' &&
                url === '//cdnjs.cloudflare.com/ajax/libs/d3/3.5.3/d3.min.js'){
                        callback()
                      }
                      var s = document.createElement('script');
                      s.src = url;
                      s.async = true;
                      s.onreadystatechange = s.onload = callback;
                      s.onerror = function(){
                        console.warn("failed to load library " + url);
                        };
                      document.getElementsByTagName("head")[0].appendChild(s);
                };
                var vincent_event = new CustomEvent(
                  "vincent_libs_loaded",
                  {bubbles: true, cancelable: true}
                );
                """
    lib_urls = [
        "'//cdnjs.cloudflare.com/ajax/libs/d3/3.5.3/d3.min.js'",
        ("'//cdnjs.cloudflare.com/ajax/libs/d3-geo-projection/0.2.9/"
         "d3.geo.projection.min.js'"),
        "'//wrobstory.github.io/d3-cloud/d3.layout.cloud.js'",
        "'//wrobstory.github.io/vega/vega.v1.3.3.js'"
    ]
    get_lib = """vct_load_lib(%s, function(){
                  %s
                  });"""
    load_js = get_lib
    ipy_trigger = "window.dispatchEvent(vincent_event);"
    for elem in lib_urls[:-1]:
        load_js = load_js % (elem, get_lib)
    load_js = load_js % (lib_urls[-1], ipy_trigger)
    html = """
           <script>
               %s
               function load_all_libs(){
                  console.log('Loading Vincent libs...')
                  %s
               };
               if(typeof define === "function" && define.amd){
                    if (window['d3'] === undefined ||
                        window['topojson'] === undefined){
                        require.config(
                            {paths: {
    d3: '//cdnjs.cloudflare.com/ajax/libs/d3/3.5.3/d3.min',
    topojson: '//cdnjs.cloudflare.com/ajax/libs/topojson/1.6.9/topojson.min'
                              }
                            }
                          );
                        require(["d3"], function(d3){
                            console.log('Loading Vincent from require.js...')
                            window.d3 = d3;
                            require(["topojson"], function(topojson){
                                window.topojson = topojson;
                                load_all_libs();
                            });
                        });
                    } else {
                        load_all_libs();
                    };
               }else{
                    console.log('Require.js not found, loading manually...')
                    load_all_libs();
               };

           </script>""" % (load_lib, load_js,)
    return display(HTML(html))

def _assert_is_type(name, value, value_type):
    """Assert that a value must be a given type."""
    if not isinstance(value, value_type):
        if type(value_type) is tuple:
            types = ', '.join(t.__name__ for t in value_type)
            raise ValueError('{0} must be one of ({1})'.format(name, types))
        else:
            raise ValueError('{0} must be {1}'
                             .format(name, value_type.__name__))

class ValidationError(Exception):
    """Exception raised with validation fails

    This exception is raised only when the ``validate`` functions of classes
    that inherit from ``FieldClass`` are called. It implies that the classes
    do not contain valid Vega JSON."""
    pass

class KeyedList(list):
    """A list that can optionally be indexed by the ``name`` attribute of
    its elements"""
    def __init__(self, attr_name='name', *args, **kwargs):
        self.attr_name = attr_name
        list.__init__(self, *args, **kwargs)

    def get_keys(self):
        keys = [getattr(x, self.attr_name) for x in self]
        if len(keys) != len(set(keys)):
            raise ValidationError('duplicate keys found')
        return keys

    def __getitem__(self, key):
        if isinstance(key, str_types):
            keys = self.get_keys()
            if key not in keys:
                raise KeyError(' "{0}" is an invalid key'.format(key))
            else:
                return self[keys.index(key)]
        else:
            return list.__getitem__(self, key)

    def __delitem__(self, key):
        if isinstance(key, str_types):
            keys = self.get_keys()
            if key not in keys:
                raise KeyError(' "{0}" is an invalid key'.format(key))
            else:
                list.__delitem__(self, keys.index(key))
        else:
            return list.__delitem__(self, key)

    def __setitem__(self, key, value):
        if isinstance(key, str_types):
            if not hasattr(value, self.attr_name):
                raise ValidationError(
                    'object must have ' + self.attr_name + ' attribute')
            elif getattr(value, self.attr_name) != key:
                raise ValidationError(
                    "key must be equal to '" + self.attr_name +
                    "' attribute")

            keys = self.get_keys()
            if key not in keys:
                self.append(value)
            else:
                list.__setitem__(self, keys.index(key), value)
        else:
            list.__setitem__(self, key, value)

def grammar(grammar_type=None, grammar_name=None):
    """Decorator to define properties that map to the ``grammar``
    dict. This dict is the canonical representation of the Vega grammar
    within Vincent.

    This decorator is intended for classes that map to some pre-defined JSON
    structure, such as axes, data, marks, scales, etc. It is assumed that this
    decorates functions with an instance of ``self.grammar``.

    Parameters
    ----------
    grammar_type : type or tuple of types, default None
        If the argument to the decorated function is not of the given types,
        then a ValueError is raised. No type checking is done if the type is
        None (default).
    grammar_name : string, default None
        An optional name to map to the internal ``grammar`` dict. If None
        (default), then the key for the dict is the name of the function
        being decorated. If not None, then it will be the name specified
        here. This is useful if the expected JSON field name is a Python
        keyword or has an un-Pythonic name.

    This should decorate a "validator" function that should return no value
    but raise an exception if the provided value is not valid Vega grammar. If
    the validator throws no exception, then the value is assigned to the
    ``grammar`` dict.

    The validator function should take only one argument - the value to be
    validated - so that no ``self`` argument is included; the validator
    should not modify the class.

    If no arguments are given, then no type-checking is done the property
    will be mapped to a field with the name of the decorated function.

    The doc string for the property is taken from the validator functions's
    doc string.
    """
    def grammar_creator(validator, name):
        def setter(self, value):
            if isinstance(grammar_type, (type, tuple)):
                _assert_is_type(validator.__name__, value, grammar_type)
            validator(value)
            self.grammar[name] = value

        def getter(self):
            return self.grammar.get(name, None)

        def deleter(self):
            if name in self.grammar:
                del self.grammar[name]

        return property(getter, setter, deleter, validator.__doc__)

    if isinstance(grammar_type, (type, tuple)):
        # If grammar_type is a type, return another decorator.
        def grammar_dec(validator):
            # Make sure to use the grammar name if it's there.
            if grammar_name:
                return grammar_creator(validator, grammar_name)
            else:
                return grammar_creator(validator, validator.__name__)
        return grammar_dec
    elif isinstance(grammar_name, str_types):
        # If grammar_name is a string, use that name and return another
        # decorator.
        def grammar_dec(validator):
            return grammar_creator(validator, grammar_name)
        return grammar_dec
    else:
        # Otherwise we assume that grammar_type is actually the function being
        # decorated.
        return grammar_creator(grammar_type, grammar_type.__name__)

class GrammarDict(dict):
    """The Vega Grammar. When called, obj.grammar returns a Python data
    structure for the Vega Grammar. When printed, obj.grammar returns a
    string representation."""

    def __init__(self, *args, **kwargs):
        """Standard Dict init"""
        dict.__init__(self, *args, **kwargs)

    def encoder(self, obj):
        """Encode grammar objects for each level of hierarchy"""
        if hasattr(obj, 'grammar'):
            return obj.grammar

    def __call__(self):
        """When called, return the Vega grammar as a Python data structure."""

        return json.loads(json.dumps(self, default=self.encoder))

    def __str__(self):
        """String representation of Vega Grammar"""

        return json.dumps(self, default=self.encoder)

class GrammarClass(object):
    """Base class for objects that rely on an internal ``grammar`` dict. This
    dict contains the complete Vega grammar.

    This should be used as a superclass for classes that map to some JSON
    structure. The JSON content is stored in an internal dict named
    ``grammar``.
    """
    def __init__(self, **kwargs):
        """Initialize a GrammarClass

        **kwargs are attribute-value pairs that are set on initialization.
        These will generally be keys for the ``grammar`` dict. If the
        attribute does not already exist as a property, then a
        ``ValueError`` is raised.
        """
        self.grammar = GrammarDict()

        for attr, value in sorted(kwargs.items()):
            if hasattr(self, attr):
                setattr(self, attr, value)
            else:
                raise ValueError('unknown keyword argument ' + attr)

    def validate(self):
        """Validate the contents of the object.

        This calls ``setattr`` for each of the class's grammar properties. It
        will catch ``ValueError``s raised by the grammar property's setters
        and re-raise them as :class:`ValidationError`.
        """
        for key, val in self.grammar.items():
            try:
                setattr(self, key, val)
            except ValueError as e:
                raise ValidationError('invalid contents: ' + e.args[0])

    def to_json(self, path=None, html_out=False,
                html_path='vega_template.html', validate=False,
                pretty_print=True):
        """Convert object to JSON

        Parameters
        ----------
        path: string, default None
            Path to write JSON out. If there is no path provided, JSON
            will be returned as a string to the console.
        html_out: boolean, default False
            If True, vincent will output an simple HTML scaffold to
            visualize the vega json output.
        html_path: string, default 'vega_template.html'
            Path for the html file (if html_out=True)
        validate : boolean
            If True, call the object's `validate` method before
            serializing. Default is False.
        pretty_print : boolean
            If True (default), JSON is printed in more-readable form with
            indentation and spaces.

        Returns
        -------
        string
            JSON serialization of the class's grammar properties.
        """
        if validate:
            self.validate()

        if pretty_print:
            dumps_args = {'indent': 2, 'separators': (',', ': ')}
        else:
            dumps_args = {}

        def encoder(obj):
            if hasattr(obj, 'grammar'):
                return obj.grammar

        if html_out:
            template = Template(
                str(resource_string('vincent', 'vega_template.html')))
            with open(html_path, 'w') as f:
                f.write(template.substitute(path=path))

        if path:
            with open(path, 'w') as f:
                json.dump(self.grammar, f, default=encoder, sort_keys=True,
                          **dumps_args)
        else:
            return json.dumps(self.grammar, default=encoder, sort_keys=True,
                              **dumps_args)

    def from_json(self):
        """Load object from JSON

        Not yet implemented.
        """
        raise NotImplementedError()

class LoadError(Exception):
    """Exception for errors on loading data from third-party objects"""
    pass

max_issues_repo_name: mikemsq/vincent

max_forks_repo_name: mikemsq/vincent


text:
# -*- coding: utf-8 -*-
"""Topic Modeling(LDA) & Word2Vec.
"""

import os
import re
import sys
import json
import zipfile
import urllib
import random
import warnings
import subprocess
import itertools as it
import functools as ft
import collections
from glob import glob
from pprint import pprint
import numpy as np
import pandas as pd

import gensim
import pyLDAvis
import pyLDAvis.gensim as gensimvis

import unidecode
from unicodedata import normalize

import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm

__all__ = []
__all__ += [
    # 'compute_coherence_values',
    # 'pick_best_topics',
    # 'groupby_top_n',
    # 'get_terminfo_table',
    'TopicModeler',
]

def compute_coherence_values(
        dictionary,
        corpus,
        id2word,
        texts,
        num_topic_list=[5, 10],
        lda_type='default',  # {'default', 'mallet'}
        workers_n=2,
        random_seed=1,
        ):
    """
    Compute c_v coherence for various number of topics

    Parameters:
    ----------
    dictionary : Gensim dictionary
    corpus : Gensim corpus
    texts : List of input texts
    limit : Max num of topics

    Returns:
    -------
    model_list : List of LDA topic models
    coherence_value_list : Coherence values corresponding to the LDA model,
        with respective number of topics
    """
    model_list = []
    coherence_list = []

    if random_seed:
        random.seed(random_seed)
        np.random.seed(random_seed)
    if lda_type == 'default':
        for num_topics in num_topic_list:
            model = gensim.models.LdaMulticore(
                corpus,
                num_topics=num_topics,
                id2word=id2word,
                passes=2,
                workers=workers_n,
                eta='symmetric',
                decay=.8,  # {.5, 1.}
                per_word_topics=False,
                offset=1.,
                iterations=30,
                gamma_threshold=.001,  # 0.001,
                minimum_probability=.05,  # .01,
                minimum_phi_value=.01,
                random_state=random_seed,
            )
            coherence_model = gensim.models.CoherenceModel(
                model=model,
                texts=texts,
                dictionary=id2word,
                coherence='c_v',
            )

            model_list += [model]
            coherence_list += [coherence_model.get_coherence()]

    elif lda_type == 'hdp':
        for num_topics in num_topic_list:
            model = gensim.models.HdpModel(
                corpus,
                id2word=id2word,
                T=3,
                # alpha=,
                K=num_topics,
                # gamma=,
                # decay=.5, # {.5, 1.}
                # per_word_topics=True,
                # minimum_probability=.1,
                # minimum_phi_value=.01,
                random_state=random_seed,
            )
            coherence_model = gensim.models.CoherenceModel(
                model=model,
                texts=texts,
                dictionary=id2word,
                coherence='c_v',
            )

            model_list += [model]
            coherence_list += [coherence_model.get_coherence()]

    elif lda_type == 'mallet':
        # Download File: http://mallet.cs.umass.edu/dist/mallet-2.0.8.zip
        mallet_url = 'http://mallet.cs.umass.edu/dist/mallet-2.0.8.zip'
        mallet_filename = mallet_url.split('/')[-1]
        mallet_unzipped_dirname = mallet_filename.split('.zip')[0]
        mallet_path = f'{mallet_unzipped_dirname}/bin/mallet'

        if not os.path.exists(mallet_path):
            # download the url contents in binary format
            urllib.request.urlretrieve(mallet_url, mallet_filename)

            # open method to open a file on your system and write the contents
            with zipfile.ZipFile(mallet_filename, "r") as zip_ref:
                zip_ref.extractall(mallet_unzipped_dirname)

        for num_topics in num_topic_list:
            model = gensim.models.wrappers.LdaMallet(
                mallet_path,
                corpus=corpus,
                num_topics=num_topics,
                id2word=id2word,
            )
            coherence_model = gensim.models.CoherenceModel(
                model=model,
                texts=texts,
                dictionary=id2word,
                coherence='c_v',
            )

            model_list += [model]
            # coherence_list += [coherence_model.get_coherence()]

    return model_list, coherence_list

# Print the coherence scores
def pick_best_topics(
        dictionary,
        corpus,
        texts,
        num_topic_list=[5, 7, 10, 12, 15, 17, 20],
        lda_type='default',
        workers_n=2,
        random_seed=1,
        ):
    model_list, coherence_value_list = compute_coherence_values(
        dictionary=dictionary,
        corpus=corpus,
        id2word=dictionary,
        texts=texts,
        num_topic_list=num_topic_list,
        lda_type=lda_type,
        workers_n=workers_n,
        random_seed=random_seed,
        #  start=2, limit=40, step=6,
    )

    paired = zip(model_list, coherence_value_list)
    ordered = sorted(paired, key=lambda x: x[1], reverse=True)
    best_model = ordered[0][0]

    model_coh_list = []
    model_topicnum_list = []
    for i, (m, cv) in enumerate(zip(model_list, coherence_value_list)):
        topic_num = m.num_topics
        coh_value = round(cv, 4)
        print(
            f'[{i}] Num Topics ({topic_num:2})' +
            f' has Coherence Value of {coh_value}'
        )
        # model_topicnum_list += [(topic_num, m)]
        model_coh_list += [(topic_num, m, coh_value)]
        model_topicnum_list += [
            (topic_num, {'model': m, 'coherence': coh_value})
        ]

    model_dict = dict(model_topicnum_list)
    print(f'Best N topics: {best_model.num_topics}')

    return best_model, model_coh_list, model_dict, coherence_value_list

def get_saliency(tinfo_df):
    r"""Calculate Saliency for terms within a topic.

    .. math::
        distinctiveness(w) = \sum P(t \vert w) log\frac{P(t \vert w)}{P(w)}
        saliency(w) = P(w) \times distinctiveness(w)

(Chuang, J., 2012. Termite: Visualization techniques for assessing textual topic models)

    Parameters
    ----------
    tinfo: pandas.DataFrame
        `pyLDAvis.gensim.prepare`.to_dict()['tinfo'] containing
        ['Category', 'Freq', 'Term', 'Total', 'loglift', 'logprob']

    Return
    ------
    saliency: float

    """

    saliency = tinfo_df['Freq'] / tinfo_df['Total']

    return saliency

def get_relevance(tinfo_df, lambda_val=.6):
    r"""Calculate Relevances with a given lambda value.

    .. math::
        relevance(t,w) = \lambda \cdot P(w \vert t) + (1 - \lambda) \cdot \frac{P(w \vert t)}{P(w)}
        Recommended \lambda = 0.6

(Sievert, C., 2014. LDAvis: A method for visualizing and interpreting topics)

    Parameters
    ----------
    tinfo: pandas.DataFrame
        `pyLDAvis.gensim.prepare`.to_dict()['tinfo'] containing
        ['Category', 'Freq', 'Term', 'Total', 'loglift', 'logprob']

    lambda_val: float
        lambda_ratio between {0-1}. default is .6 (recommended from its paper)

    Return
    ------
    relevance: float

    """

    relevance = l * tinfo_df['logprob'] + (1 - l) * tinfo_df['loglift']

    return relevance

def groupby_top_n(
        dataframe,
        group_by=None,
        order_by=None,
        ascending=False,
        n=5,
        ):

    res_df = (
        dataframe
        .groupby(group_by)
        [dataframe.columns.drop(group_by)]
        .apply(
            lambda x: x.sort_values(order_by, ascending=ascending).head(n)
        )
    )
    return res_df

def _df_with_names(data, index_name, columns_name):
    """A renaming function from `pyLDAvis._prepare`.
    """
    if type(data) == pd.DataFrame:
        # we want our index to be numbered
        df = pd.DataFrame(data.values)
    else:
        df = pd.DataFrame(data)
    df.index.name = index_name
    df.columns.name = columns_name
    return df

def _series_with_name(data, name):
    """A renaming function from `pyLDAvis._prepare`.
    """
    if type(data) == pd.Series:
        data.name = name
        # ensures a numeric index
        return data.reset_index()[name]
    else:
        return pd.Series(data, name=name)

def get_terminfo_table(
        lda_model,
        corpus: list = None,
        dictionary: gensim.corpora.dictionary.Dictionary = None,
        doc_topic_dists=None,
        use_gensim_prepared=True,
        top_n=10,
        workers_n=-1,
        r_normalized=False,
        relevence_lambda_val=.6,
        random_seed=1,
        ):

    if random_seed:
        random.seed(random_seed)
        np.random.seed(random_seed)

    if use_gensim_prepared:
        _prepared = gensimvis.prepare(
            topic_model=lda_model,
            corpus=corpus,
            dictionary=dictionary,
            doc_topic_dist=None,
            R=len(dictionary),
            # lambda_step=0.2,
            mds='tsne',
            # mds=<function js_PCoA>,
            n_jobs=workers_n,
            plot_opts={'xlab': 'PC1', 'ylab': 'PC2'},
            sort_topics=True,
        )
        tinfo_df = pd.DataFrame(_prepared.to_dict()['tinfo'])

        tinfo_df['topic_term_dists'] = np.exp(tinfo_df['logprob'])
        tinfo_df['term_proportion'] = (
            np.exp(tinfo_df['logprob']) / np.exp(tinfo_df['loglift'])
        )
        tinfo_df['saliency'] = get_saliency(tinfo_df)
        tinfo_df['relevance'] = get_relevance(
            tinfo_df,
            lambda_val=relevence_lambda_val,
        )

        tinfo_df['term_prob'] = np.exp(tinfo_df['logprob'])
        tinfo_df['term_r_prob'] = np.exp(tinfo_df['relevance'])
        tinfo_df['term_r_adj_prob'] = (
            tinfo_df
            .groupby(['Category'])
            ['term_r_prob']
            .apply(lambda x: x / x.sum())
        )

        if r_normalized:
            r_colname = 'term_r_adj_prob'
        else:
            r_colname = 'term_r_prob'

        relevance_score_df = (
            tinfo_df[tinfo_df['Category'] != 'Default']
            .groupby(['Category', 'Term'])
            [[r_colname]]
            .sum()
            .reset_index()
        )

        # corpus_dict_df = pd.DataFrame(
        #     # It is possible
        #     # because the keys of this dictionary generated from range(int).
        #     # Usually the dictionary is iterable but not ordered.
        #     list(dictionary.values()),
        #     # [dictionary[i] for i, _ in enumerate(dictionary)],
        #     columns=['Term'],
        # )
        # corpus_dict_df['term_id'] = corpus_dict_df.index
        corpus_dict_df = pd.DataFrame(
            list(dictionary.items()),
            columns=['term_id', 'Term'],
        )
        corpus_dict_df.set_index('term_id', drop=False, inplace=True)

        r_score_df = pd.merge(
            relevance_score_df,
            corpus_dict_df,
            on=['Term'],
            how='left',
        )
        r_score_df['category_num'] = (
            r_score_df['Category']
            .str
            .replace('Topic', '')
            .astype(int) - 1
        ).astype('category')
        r_score_df.set_index(['category_num', 'term_id'], inplace=True)
        ixs = pd.IndexSlice

        topic_list = r_score_df.index.levels[0]
        equal_prob = 1. / len(topic_list)
        empty_bow_case_list = list(
            zip(topic_list, [equal_prob] * len(topic_list))
        )

        def get_bow_score(
                bow_chunk,
                score_df=r_score_df,
                colname=r_colname,
                ):

            bow_chunk_arr = np.array(bow_chunk)
            word_id_arr = bow_chunk_arr[:, 0]
            word_cnt_arr = bow_chunk_arr[:, 1]

            # normed_word_cnt_arr = (word_cnt_arr / word_cnt_arr.sum()) * 10
            clipped_word_cnt_arr = np.clip(word_cnt_arr, 0, 3)

            score_series = (score_df.loc[ixs[:, word_id_arr], :]
                            .groupby(level=0)
                            [colname]
                            .apply(lambda x: x @ clipped_word_cnt_arr)
                            )
            score_list = list(score_series.iteritems())
            # normed_score_series = score_series / score_series.sum()
            # score_list = list(normed_score_series.iteritems())

            return score_list

        bow_score_list = [
            get_bow_score(bow_chunk)
            if bow_chunk not in (None, [])
            else empty_bow_case_list
            for bow_chunk in corpus
        ]

        relevant_terms_df = groupby_top_n(
            tinfo_df,
            group_by=['Category'],
            order_by=['relevance'],
            ascending=False,
            n=top_n,
        )
        relevant_terms_df['rank'] = (
            relevant_terms_df
            .groupby(['Category'])
            ['relevance']
            # .rank(method='max')
            .rank(method='max', ascending=False)
            .astype(int)
        )

    else:
        vis_attr_dict = gensimvis._extract_data(
            topic_model=lda_model,
            corpus=corpus,
            dictionary=dictionary,
            doc_topic_dists=None,
        )
        topic_term_dists = _df_with_names(
            vis_attr_dict['topic_term_dists'],
            'topic', 'term',
        )
        doc_topic_dists = _df_with_names(
            vis_attr_dict['doc_topic_dists'],
            'doc', 'topic',
        )
        term_frequency = _series_with_name(
            vis_attr_dict['term_frequency'],
            'term_frequency',
        )
        doc_lengths = _series_with_name(
            vis_attr_dict['doc_lengths'],
            'doc_length',
        )
        vocab = _series_with_name(
            vis_attr_dict['vocab'],
            'vocab',
        )

        ## Topic
        # doc_lengths @ doc_topic_dists
        topic_freq = (doc_topic_dists.T * doc_lengths).T.sum()
        topic_proportion = (topic_freq / topic_freq.sum())

        ## reorder all data based on new ordering of topics
        # topic_proportion = (topic_freq / topic_freq.sum()).sort_values(ascending=False)
        # topic_order = topic_proportion.index
        # topic_freq = topic_freq[topic_order]
        # topic_term_dists = topic_term_dists.iloc[topic_order]
        # doc_topic_dists = doc_topic_dists[topic_order]

        # token counts for each term-topic combination
        term_topic_freq = (topic_term_dists.T * topic_freq).T
        term_frequency = np.sum(term_topic_freq, axis=0)

        ## Term
        term_proportion = term_frequency / term_frequency.sum()

        # compute the distinctiveness and saliency of the terms
        topic_given_term = topic_term_dists / topic_term_dists.sum()
        kernel = (topic_given_term *
                  np.log((topic_given_term.T / topic_proportion).T))
        distinctiveness = kernel.sum()
        saliency = term_proportion * distinctiveness

        default_tinfo_df = pd.DataFrame(
            {
                'saliency': saliency,
                'term': vocab,
                'freq': term_frequency,
                'total': term_frequency,
                'category': 'default',
                'logprob': np.arange(len(vocab), 0, -1),
                'loglift': np.arange(len(vocab), 0, -1),
            }
        )

        log_lift = np.log(topic_term_dists / term_proportion)
        log_prob = log_ttd = np.log(topic_term_dists)

    return _prepared, tinfo_df, relevant_terms_df, r_score_df, bow_score_list

class TopicModeler(object):
    """Topic Modeling via LDA(Latent Diriclet Allocation).

    Get tokenized from text.

    Parameters
    ----------

    sentence_list: list
        A list of raw sentences.

    tokenized_sentence_list: list
        A nested list of tokenized sentences.

    Attributes
    ----------

    After `__init__`:
        self.sentences:
            A list of raw sentences.
        self.tokenized: list
            A nested list of tokenized sentences.

        self.corpora_dict: `gensim.corpora.dictionary.Dictionary`
            A token dictionary from a given text.

        self.bow_corpus_idx: list
            A nested list, which contains converted documents into a list of token indices.

        self.bow_corpus_doc: list
            A nested list, which contains converted documents into a list of token words.

    After `train_lda` or `load_lda`:
        self.best_lda_model: dict
            A dict contains the best model & its coherence value.
            `{'coherence': int, 'model':gensim.models.ldamulticore.LdaMulticore}`

        self.lda_model_list = model_coh_list
            A nested list of `[topic_num, model, coherence_value]`

        self.lda_model_dict:
            A nested dict as `{topic_num: {'coherence': int, 'model': `gensim.models.ldamulticore.LdaMulticore`}}`

        self.trained: bool
            `True` If trained or properly loaded.

    After `visualize_lda_to_html`:
        self.selected_topic_num: int
            A int of selected topic number.

        self.selected_model: `gensim.models.ldamulticore.LdaMulticore`

        self.vis_prepared: `pyLDAvis.prepared_data.PreparedData`

        self.total_terms_df
            `tinfo_table`, `'Default'` removed.

        self.top_relevant_terms_df: `pandas.DataFrame`
            A rank table of `Category`.

        self.r_adj_score_df: `pandas.DataFrame`
            A tinfo table, considering saliency and relevence score.

        self.bow_score_list: list
            Scores of each sentence, based on bow_corpus, clipped by (0, 3).

    After `estimate_topics_by_documents` or `load_estimated`:
        self.dominant_topic_estimation_df: `pandas.DataFrame`
            A dataframe contains `['lda_prob', 'dominant_topic', 'contribution', 'topic_keywords']`

        self.topic_freq_df: `pandas.DataFrame`
            A rank table by topic frequency.

    After `get_representitive_documents` or `load_representitive_documents`:
        self.representitive_docs: `pandas.DataFrame`

    After `get_representitive_candidates`:
        return `repr_sentences, repr_bow_corpus_doc, repr_bow_corpus_idx`

    Methods
    -------

    train_lda

    save_lda

    load_lda

    pick_best_lda_topics

    visualize_lda_to_html

    estimate_topics_by_documents

    get_representitive_documents

    See Also
    --------
    Preprocessing
        ``unipy_nlp.preprocessing.Preprocessor``

    POS-Tagging
        ``konlpy.tag.Mecab``

    Byte-Pair Encoding
        ``sentencepiece``

    Examples
    --------

    >>> import unipy_nlp.data_collector as udcl
    >>> import unipy_nlp.preprocessing as uprc
    >>> import unipy_nlp.analyze.topic_modeling as utpm
    >>> from pprint import pprint
    >>> prep = uprc.Preprocessor()
    >>> prep.read_json('./data/_tmp_dump/prep/rawdata_collected.json')
    >>> sentence_for_pos_list = [
    ...     "무궁화 꽃이 피었습니다."
    ...     "우리는 민족중흥의 역사적 사명을 띠고 이 땅에 태어났다.",
    ... ]
    >>> tokenized = prep.pos_tag(
    ...     input_text=sentence_for_pos_list,
    ...     tag_type=[
    ...         '체언 접두사', '명사', '한자', '외국어',
    ...         '수사', '구분자',
    ...         '동사',
    ...         '부정 지정사', '긍정 지정사',
    ...     ]
    ... )
    >>> print(tokenized)
    [['무궁화'], ['우리', '민족중흥', '역사', '사명']]
    >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
    >>> tpm.train_lda(
    ...     num_topic=5,
    ...     workers_n=8,
    ...     random_seed=1,
    ... )
    >>> tpm.save_lda(savepath='data/_tmp_dump/topic_modeling', affix='lda')
    >>> tpm.load_lda('data/_tmp_dump/topic_modeling')
    >>> tpm.pick_best_lda_topics(
    ...     num_topic_list=[5, 7, 10],
    ...     workers_n=8,
    ...     random_seed=1,
    ... )
    >>> tpm.visualize_lda_to_html(
    ...     7,
    ...     top_n=10,
    ...     r_normalized=False,
    ...     relevence_lambda_val=.6,
    ...     workers_n=8,
    ...     random_seed=1,
    ...     savepath='data/_tmp_dump/topic_modeling',
    ...     filename_affix='lda',
    ...     # save_type='html',  # {'html', 'json'}
    ...     save_relevent_terms_ok=True,
    ...     save_html_ok=True,
    ...     display_ok=False,
    ... )

    >>> sentence_labeled = tpm.estimate_topics_by_documents(
    ...     7,
    ...     # sentence_list=tokenized,
    ...     random_seed=1,
    ...     save_ok=True,
    ...     savepath='data/_tmp_dump/topic_modeling',
    ...     filename_affix='lda',
    ... )
    >>> sentence_repr = tpm.get_representitive_documents(
    ...     7,
    ...     len_range=(10, 30),
    ...     top_n=10,
    ...     save_ok=True,
    ...     savepath='data/_tmp_dump/topic_modeling',
    ...     filename_affix='lda',
    ... )

    """
    def __init__(
            self,
            sentence_list,
            tokenized_sentence_list,
            ):

        self.trained = False  # {'trained', 'loaded'}
        self.selected_model = None
        self.lda_model_list = None

        self.sentences = sentence_list
        self.tokenized = tokenized_sentence_list
        self.corpora_dict = gensim.corpora.Dictionary(
            tokenized_sentence_list
        )
        self.corpora_dict.filter_extremes(
            no_below=30, no_above=.5, keep_n=100000,
        )

        self.bow_corpus_idx = [
            self.corpora_dict.doc2idx(doc)
            for doc in tokenized_sentence_list
        ]
        self.bow_corpus_doc = [
            self.corpora_dict.doc2bow(doc)
            for doc in tokenized_sentence_list
        ]

    def train_lda(
            self,
            num_topic=5,
            lda_type='default',
            workers_n=2,
            random_seed=1,
            ):
        """
        Train a single LDA Topic Model.

        Parameters
        ----------

        num_topics: int (default: 5)
            A number of topics.

        lda_type: str (default: `'default'`, `{'default', 'hdp', 'mallet'}`)
            A type of LDA model.
            Use `'default'` for now. Other options are working in progress.

        workers_n: int (default: 2)
            A number of CPU core to train.

        random_seed: int (default: 1)
            A random seed int.

        Example
        -------

        >>> import unipy_nlp.data_collector as udcl
        >>> import unipy_nlp.preprocessing as uprc
        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> from pprint import pprint
        >>> prep = uprc.Preprocessor()
        >>> prep.read_json('./data/_tmp_dump/prep/rawdata_collected.json')
        >>> sentence_for_pos_list = [
        ...     "무궁화 꽃이 피었습니다."
        ...     "우리는 민족중흥의 역사적 사명을 띠고 이 땅에 태어났다.",
        ... ]
        >>> tokenized = prep.pos_tag(
        ...     input_text=sentence_for_pos_list,
        ...     tag_type=[
        ...         '체언 접두사', '명사', '한자', '외국어',
        ...         '수사', '구분자',
        ...         '동사',
        ...         '부정 지정사', '긍정 지정사',
        ...     ]
        ... )
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.train_lda(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )

        """
        self.pick_best_lda_topics(
            num_topic_list=[num_topic],
            lda_type=lda_type,
            workers_n=workers_n,
            random_seed=random_seed,
        )
        self.trained = True

    def pick_best_lda_topics(
            self,
            num_topic_list=[5, 7, 10, 12, 15, 17, 20],
            lda_type='default',
            workers_n=2,
            random_seed=1,
            ):
        """
        Train multiple LDA Topic Models by given topic numbers.

        Parameters
        ----------

        num_topic_list: list (default: `[5, 7, 10, 12, 15, 17, 20]`)
            A list of topic numbers.

        lda_type: str (default: `'default'`, `{'default', 'hdp', 'mallet'}`)
            A type of LDA model.
            Use `'default'` for now. Other options are working in progress.

        workers_n: int (default: 2)
            A number of CPU core to train.

        random_seed: int (default: 1)
            A random seed int.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )

        """

        (best_lda_model, model_coh_list,
         model_dict, coh_list) = pick_best_topics(
            dictionary=self.corpora_dict,
            corpus=self.bow_corpus_doc,
            texts=self.tokenized,
            num_topic_list=num_topic_list,
            lda_type=lda_type,
            workers_n=workers_n,
            random_seed=random_seed,
        )

        self.best_lda_model = best_lda_model
        self.lda_model_list = model_coh_list
        self.lda_model_dict = model_dict

        self.trained = True

    def save_lda(self, savepath='./', affix='lda'):
        """
        Save trained lda model(s).

        Parameters
        ----------

        savepath: str (default: `'./'`)
            A dirpath to save.

        affix: str (default: `'lda'`)
            An affix of filename. Its ext will be `.ldamodel`.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> tpm.save_lda(savepath='data/_tmp_dump/topic_modeling', affix='lda')

        """
        os.makedirs(savepath, exist_ok=True)

        corpora_filename = os.path.join(
            savepath,
            f'{affix}.cdict',
        )
        self.corpora_dict.save_as_text(
            corpora_filename,
            sort_by_word=False,
        )

        for _topic_num, _inner_dict in self.lda_model_dict.items():

            _model = _inner_dict['model']
            _coh_val = _inner_dict['coherence']

            model_name_str = '_'.join([
                f'{affix}',
                f'topics-{_topic_num}',
                f'coh-{_coh_val}.ldamodel',
            ])
            print(f'{savepath:2}: {model_name_str}')

            _filename = os.path.join(
                savepath,
                model_name_str,
            )
            _model.save(_filename)

    def load_lda(self, filepath):
        """
        Load trained lda model(s).

        Parameters
        ----------

        filepath: str
            A dirpath to load. It contains `.ldamodel`.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.load_lda('data/_tmp_dump/topic_modeling')

        """
        if os.path.isfile(filepath):

            self.lda_model_list = []
            self.lda_model_dict = {}

            path_str = '/'.join(filepath.split('/')[:-1])
            model_name_str = filepath.split('/')[-1]
            affix, _topic_num, _coh_val = re.findall(
                r'^(.+)_topics\-(\d+)_coh\-([-+]?\d*\.\d+|\d+)\.ldamodel',
                model_name_str,
            )[0]

            _topic_num, _coh_val = int(_topic_num), int(_coh_val)
            _model = gensim.models.LdaMulticore.load(filepath)
            self.lda_model_list = [(_topic_num, _model, _coh_val)]
            self.lda_model_dict.__setitem__(
                _topic_num,
                {'model': _model, 'coherence': _coh_val},
            )

            corpora_filename = os.path.join(
                path_str,
                f'{affix}.cdict',
            )
            self.corpora_dict = gensim.corpora.Dictionary.load_from_text(
                corpora_filename,
            )
            self.best_lda_model = self.lda_model_dict[_topic_num]
            print(f'Model loaded: topics={_topic_num}, coh={_coh_val}')

        elif os.path.isdir(filepath):

            self.lda_model_list = []
            self.lda_model_dict = {}

            path_str = filepath
            filepath_list = glob(os.path.join(filepath, '*.ldamodel'))

            for filename in filepath_list:
                model_name_str = filename.split('/')[-1]
                affix, _topic_num, _coh_val = re.findall(
                    r'^(.+)_topics\-(\d+)_coh\-([-+]?\d*\.\d+|\d+).ldamodel',
                    model_name_str,
                )[0]
                _topic_num, _coh_val = int(_topic_num), int(_coh_val)
                _model = gensim.models.LdaMulticore.load(filename)
                self.lda_model_list += [(_topic_num, _model, _coh_val)]
                self.lda_model_dict.__setitem__(
                    _topic_num,
                    {'model': _model, 'coherence': _coh_val},
                )
                print(f'Model loaded: topics={_topic_num}, coh={_coh_val}')

            corpora_filename = os.path.join(
                path_str,
                f'{affix}.cdict',
            )
            self.corpora_dict = gensim.corpora.Dictionary.load_from_text(
                corpora_filename,
            )
            best_topic_num, best_model, best_coh_val = max(
                self.lda_model_list,
                key=lambda x: x[-1],
            )
            self.best_lda_model = self.lda_model_dict[int(best_topic_num)]

        self.trained = True

    def _get_terminfo_table(self, *args, **kwargs):
        return get_terminfo_table(*args, **kwargs)

    def visualize_lda_to_html(
            self,
            target_topic_num,
            top_n=10,
            r_normalized=False,
            relevence_lambda_val=.6,
            workers_n=2,
            random_seed=1,
            savepath='./',
            filename_affix='lda',
            # save_type='html',  # {'html', 'json'}
            save_relevent_terms_ok=True,
            save_html_ok=True,
            display_ok=False,
            ):
        """
        Run `pyLDAvis.prepare` & get adjusted scores(use saliency & relevence) of terms by each topic.

        Parameters
        ----------

        target_topic_num: int
            A topic number of LDA model to visualize.

        top_n: int (default: `10`)
            A number of the most relevent terms in a topic.

        r_normalized: bool (default: `False`)
            Use normalized probabilities when it is `True`. (not recommended in most cases.)

        relevence_lambda_val: float (defautl: `.6`).
            A lambda value(ratio) to calculate relevence.

        workers_n: int (default: `2`)
            A number of CPU cores to calculate(`pyLDAvis.prepare`)

        random_seed: int (default: `1`)
            A random seed number.

        savepath: str (default: `'./'`)
            A dirpath to save `pyLDAvis` or other `pandas.DataFrame`s.

        filename_affix: str (default: `'lda'`)
            An affix of filename to save `pyLDAvis` html or json.

        save_relevent_terms_ok: bool (default: `True`)
            An option to save `pandas.DataFrame` of `top_relevent_terms`.

        save_html_ok: bool (default: `True`)
            An option to save html.

        display_ok: bool (default: `False`)
            Call `pyLDAvis.display` when it is `True`.

        References
        ----------

        Saliency:
            `Chuang, J., 2012. Termite: Visualization techniques for assessing textual topic models`

        Relevence:
            `Sievert, C., 2014. LDAvis: A method for visualizing and interpreting topics`

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> tpm.visualize_lda_to_html(
        ...     7,
        ...     top_n=10,
        ...     r_normalized=False,
        ...     relevence_lambda_val=.6,
        ...     workers_n=8,
        ...     random_seed=1,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ...     save_relevent_terms_ok=True,
        ...     save_html_ok=True,
        ...     display_ok=False,
        ... )

        """
        if target_topic_num in self.lda_model_dict.keys():
            self.selected_topic_num = target_topic_num
            self.selected_model = (
                self.lda_model_dict[target_topic_num]['model']
            )
        else:
            raise KeyError("Model doesn't exist. Select a proper number.")

        (vis_prepared,
         total_terms_df,
         top_relevant_terms_df,
         r_adj_score_df,
         bow_score_list) = self._get_terminfo_table(
            self.selected_model,
            corpus=self.bow_corpus_doc,
            dictionary=self.corpora_dict,
            doc_topic_dists=None,
            use_gensim_prepared=True,
            top_n=top_n,
            r_normalized=r_normalized,
            relevence_lambda_val=relevence_lambda_val,
            workers_n=workers_n,
            random_seed=random_seed,
        )

        self.vis_prepared = vis_prepared
        self.total_terms_df = total_terms_df
        self.top_relevant_terms_df = top_relevant_terms_df
        self.r_adj_score_df = r_adj_score_df
        self.bow_score_list = bow_score_list

        if save_html_ok:
            os.makedirs(savepath, exist_ok=True)
            ldavis_filename_html_str = os.path.join(
                savepath,
                f'{filename_affix}_topics-{target_topic_num}.html',
            )
            pyLDAvis.save_html(
                self.vis_prepared,
                ldavis_filename_html_str,
            )
            print(f"LDAVIS HTML Saved: '{ldavis_filename_html_str}'")

        if save_relevent_terms_ok:
            os.makedirs(savepath, exist_ok=True)
            ldavis_filename_rdf_str = os.path.join(
                savepath,
                '_'.join([
                    f'{filename_affix}',
                    f'topics-{target_topic_num}',
                    f'top{top_n}_relevent_terms_df.csv',
                ]),
            )
            self.top_relevant_terms_df.to_csv(
                ldavis_filename_rdf_str,
                index=True,
                header=True,
                encoding='utf-8',
            )
            print(f"LDAVIS DF Saved: '{ldavis_filename_rdf_str}'")

        if display_ok:
            pyLDAvis.display(self.vis_prepared, local=False)

    def estimate_topics_by_documents(
            self,
            target_topic_num,
            # sentence_list=None,
            random_seed=1,
            save_ok=True,
            savepath='./',
            filename_affix='lda',
            ):
        """
        Get dominant topics & its contribution scores from each documents.

        Parameters
        ----------

        target_topic_num: int
            A topic number of LDA model to use.

        random_seed: int (default: `1`)
            A random seed number.

        save_ok: bool (default: `True`)
            Save return `pandas.DataFrame`.

        savepath: str (default: `'./'`)
            A dirpath to save the topic-labeled sentences.

        filename_affix: str (default: `'lda'`)
            An affix of filename to save the topic-labeled sentences.

        Return
        ------

        dominant_topic_estimation_df: `pandas.DataFrame`
            Topic-labeled given(trained) sentences.

        topic_freq_df: `pandas.DataFrame`
            A rank table of topics by frequency.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> sentence_labeled = tpm.estimate_topics_by_documents(
        ...     7,
        ...     random_seed=1,
        ...     save_ok=True,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )

        """

        if target_topic_num != self.selected_topic_num:
            raise ValueError(
                'You should run `visualize_lda_to_html` first.'
            )

        lda_model = self.selected_model
        corpus = self.bow_corpus_doc
        docs = self.sentences
        bow_r_score_list = self.bow_score_list
        top_r_terms_df = self.r_adj_score_df

        res_df = pd.DataFrame(
            columns=[
                'dominant_topic',
                'contribution',
                'topic_keywords',
                'documents',
                'lda_prob',
            ]
        )

        if random_seed:
            random.seed(random_seed)
            np.random.seed(random_seed)

        r_colname = top_r_terms_df.columns.drop(['Category', 'Term'])[0]

        if top_r_terms_df is not None:
            top_sorted_words = groupby_top_n(
                top_r_terms_df.reset_index(),
                group_by=['category_num'],
                order_by=[r_colname],
                ascending=False,
                n=10,
            )
            top_word_str = (
                top_sorted_words
                .groupby(level=0)
                ['Term']
                .apply(lambda x: ', '.join(x.tolist()))
            )

        def normalize_prob(prob_row):
            total_prob = sum([prob for topic, prob in prob_row])
            normed_prob_row = [
                (topic, prob / total_prob) for topic, prob in prob_row
            ]
            return normed_prob_row

        def sort_prob(prob_row, bow_r_score_list=bow_r_score_list):
            return sorted(prob_row, key=lambda x: x[1], reverse=True)

        def get_dominant_prob(prob_row):
            return pd.Series(prob_row[0])

        def get_topic_keywords(
                dom_topic_num,
                lda_model=lda_model,
                top_r_terms_df=top_r_terms_df,
                ):
            if top_r_terms_df is not None:
                return top_word_str[int(dom_topic_num)]
            else:
                return ', '.join(
                    np.array(lda_model.show_topic(int(dom_topic_num)))[:, 0]
                )

        res_df['documents'] = docs

        if bow_r_score_list is not None:
            # bow_score_series = (
            #     pd.Series(bow_r_score_list).apply(normalize_prob)
            # )
            bow_score_series = pd.Series(bow_r_score_list)
        else:
            bow_score_series = pd.Series(lda_model[corpus])

        res_df['lda_prob'] = bow_score_series.apply(sort_prob)
        res_df[['dominant_topic', 'contribution']] = (
            res_df['lda_prob']
            .apply(get_dominant_prob)
        )
        res_df['dominant_topic'] = (
            res_df['dominant_topic']
            .astype(int)
            .astype('category')
        )
        res_df['topic_keywords'] = (
            res_df['dominant_topic']
            .apply(get_topic_keywords)
        )
        res_df['lda_prob'] = res_df['lda_prob'].apply(dict)
        res_df.index.name = 'doc_num'
        res_df.reset_index(inplace=True)

        self.dominant_topic_estimation_df = res_df
        self.topic_freq_df = (
            self.dominant_topic_estimation_df
            .groupby('dominant_topic')
            ['doc_num']
            .count()
            .reset_index()
            .sort_values('doc_num', ascending=False)
        )

        if save_ok:
            os.makedirs(savepath, exist_ok=True)
            filename_str = os.path.join(
                savepath,
                '_'.join([
                    f'{filename_affix}',
                    f'topics-{target_topic_num}',
                    f'dominant_topic_estimation_df.csv',
                ]),
            )
            res_df.to_csv(
                filename_str,
                index=False,
                header=True,
                encoding='utf-8',
            )
        return self.dominant_topic_estimation_df, self.topic_freq_df

    def load_estimated(
            self,
            target_topic_num,
            savepath='./',
            filename_affix='lda'
            ):
        """
        Load the result of `self.estimate_topics_by_documents`.

        Parameters
        ----------

        target_topic_num: int
            A topic number of LDA model to use.

        savepath: str (default: `'./'`)
            A dirpath to load the topic-labeled sentences.

        filename_affix: str (default: `'lda'`)
            An affix of filename to load the topic-labeled sentences.

        Return
        ------

        dominant_topic_estimation_df: `pandas.DataFrame`
            Topic-labeled given(trained) sentences.

        topic_freq_df: `pandas.DataFrame`
            A rank table of topics by frequency.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> sentence_labeled = tpm.estimate_topics_by_documents(
        ...     7,
        ...     random_seed=1,
        ...     save_ok=True,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )
        >>> sentence_labeled, topic_freq = tpm.load_estimated(
        ...     target_topic_num=7,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )

        """
        filename_str = os.path.join(
            savepath,
            '_'.join([
                f'{filename_affix}',
                f'topics-{target_topic_num}',
                f'dominant_topic_estimation_df.csv',
            ]),
        )
        res_df = pd.read_csv(
            filename_str,
            encoding='utf-8',
        )
        self.dominant_topic_estimation_df = res_df
        self.topic_freq_df = (
            self.dominant_topic_estimation_df
            .groupby('dominant_topic')
            ['doc_num']
            .count()
            .reset_index()
            .sort_values('doc_num', ascending=False)
        )

        return self.dominant_topic_estimation_df, self.topic_freq_df

    def get_best_n_terms(self):
        pass

    def get_representitive_candidates(
            self,
            len_range=(10, 30),
            ):
        """
        Get representitive candidates by length. It is for to use `unipy_nlp.network_plot`.

        Parameters
        ----------

        len_range: `list` or `tuple` (default: `(10, 30)`)
            A candidate threshold by length.

        Return
        ------

        repr_sentences: `list`
            A list of sentences.

        repr_bow_corpus_doc: `list`
            A nested list, which contains converted documents into a list of token words.

        repr_bow_corpus_idx: `list`
            A nested list, which contains converted documents into a list of token indices..

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> sentence_labeled = tpm.estimate_topics_by_documents(
        ...     7,
        ...     random_seed=1,
        ...     save_ok=True,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )
        >>> (repr_sentenced,
        >>>  repr_bow_corpus_doc,
        >>>  repr_bow_corpus_idx) = tpm.get_representitive_candidates(
        ...     len_range=(12, 30),
        ... )

        """
        len_min, len_max = len_range
        bool_mask = mask_to_filter_document_by_len = list(
            map(lambda x: len_min <= len(x) < len_max, self.sentences)
        )

        repr_bow_corpus_idx = list(
            it.compress(self.bow_corpus_idx, bool_mask)
        )
        repr_bow_corpus_doc = list(
            it.compress(self.bow_corpus_doc, bool_mask)
        )
        repr_sentences = list(
            it.compress(self.sentences, bool_mask)
        )
        return repr_sentences, repr_bow_corpus_doc, repr_bow_corpus_idx

    def _clip_document_len(
            self,
            topic_kwd_df,
            len_range=(10, 30),
            ):

        len_min, len_max = len_range
        len_series = topic_kwd_df['documents'].apply(len)
        # mask = np.where((len_series >= 100) & (len_series < 300))
        res = topic_kwd_df.loc[
            (len_series >= len_min) & (len_series < len_max),
            :
        ]
        return res

    def get_representitive_documents(
            self,
            target_topic_num,
            len_range=(10, 30),
            top_n=10,
            save_ok=True,
            savepath='./',
            filename_affix='lda',
            ):
        """
        List-up the most representitive documents by topic.

        Parameters
        ----------

        target_topic_num: int
            A topic number of LDA model to use.

        len_range: `list` or `tuple` (default: `(10, 30)`)
            A candidate threshold by length.

        top_n: int (default: `10`)
            A document number to list-up, by topic.

        save_ok: bool (default: `True`)
            An option to save.

        savepath: str (default: `'./'`)
            A dirpath to load the topic-labeled sentences.

        filename_affix: str (default: `'lda'`)
            An affix of filename to load the topic-labeled sentences.

        Return
        ------

        reordered: `pandas.DataFrame`
            Representitive documents, group by topic, ordery by its rank.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> sentence_labeled = tpm.estimate_topics_by_documents(
        ...     7,
        ...     random_seed=1,
        ...     save_ok=True,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )
        >>> sentence_repr = tpm.get_representitive_documents(
        ...     7,
        ...     len_range=(10, 30),
        ...     top_n=10,
        ...     save_ok=True,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )

        """
        if target_topic_num != self.selected_topic_num:
            raise ValueError(
                'You should run `visualize_lda_to_html` first.'
            )

        if self.dominant_topic_estimation_df is None:
            raise ValueError(
                "You should run `estimate_topics_by_documents` first."
            )

        target_topic_num = len(self.topic_freq_df)
        repr_candidates_df = self._clip_document_len(
            self.dominant_topic_estimation_df,
            len_range=len_range,
        )

        repr_docs_df = groupby_top_n(
            repr_candidates_df,
            group_by=['dominant_topic'],
            order_by=['contribution'],
            ascending=False,
            n=top_n,
        )

        grouped_dict = dict(list(repr_docs_df.groupby(level=0)))
        reordered = pd.concat(
            [
                grouped_dict[i]
                for i in self.topic_freq_df['dominant_topic']
            ]
        )
        self.representitive_docs = reordered

        if save_ok:
            os.makedirs(savepath, exist_ok=True)
            filename_str = os.path.join(
                savepath,
                '_'.join([
                    f'{filename_affix}',
                    f'topics-{target_topic_num}',
                    f'top{top_n}_repr_docs_df.csv',
                ]),
            )
            reordered.to_csv(
                filename_str,
                index=True,
                header=True,
                encoding='utf-8',
            )

        return reordered

    def load_representitive_documents(
            self,
            target_topic_num,
            top_n=10,
            savepath='./',
            filename_affix='lda',
            ):
        """
        Load the result of `self.get_representitive_documents`.

        Parameters
        ----------

        target_topic_num: int
            A topic number of LDA model to use.

        top_n: int (default: `10`)
            A document number to list-up, by topic.
            The upper bound depends on how many documents saved.

        savepath: str (default: `'./'`)
            A dirpath to load the topic-labeled sentences.

        filename_affix: str (default: `'lda'`)
            An affix of filename to load the topic-labeled sentences.

        Return
        ------

        dominant_topic_estimation_df: `pandas.DataFrame`
            Topic-labeled given(trained) sentences.

        topic_freq_df: `pandas.DataFrame`
            A rank table of topics by frequency.

        Example
        -------

        >>> import unipy_nlp.analyze.topic_modeling as utpm
        >>> tpm = utpm.TopicModeler(sentence_list, tokenized)
        >>> tpm.pick_best_lda_topics(
        ...     num_topic=5,
        ...     workers_n=8,
        ...     random_seed=1,
        ... )
        >>> sentence_labeled = tpm.estimate_topics_by_documents(
        ...     7,
        ...     random_seed=1,
        ...     save_ok=True,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )
        >>> sentence_labeled, topic_freq = tpm.load_estimated(
        ...     target_topic_num=7,
        ...     savepath='data/_tmp_dump/topic_modeling',
        ...     filename_affix='lda',
        ... )

        """
        filename_str = os.path.join(
            savepath,
            '_'.join([
                f'{filename_affix}',
                f'topics-{target_topic_num}',
                f'top{top_n}_repr_docs_df.csv',
            ]),
        )
        self.representitive_docs = pd.read_csv(
            filename_str,
            encoding='utf-8',
        )

        return self.representitive_docs


text:
# !/usr/bin/env python
"""Run a test calculation on localhost.

Usage: ./example_base.py
"""
from os import path
from aiida_shengbte import helpers
from aiida import cmdline, engine, orm
from aiida.plugins import DataFactory, CalculationFactory
import click
import logging
logging.basicConfig(level=logging.INFO)

INPUT_DIR = path.join(path.dirname(path.realpath(__file__)), 'input_files')

def test_run(shengbte_code):
    """Run a calculation on the localhost computer.

    Uses test helpers to create AiiDA Code on the fly.
    """
    if not shengbte_code:
        # get code
        computer = helpers.get_computer()
        shengbte_code = helpers.get_code(entry_point='shengbte',
                                         computer=computer,
                                         prepend_text='find _* |')
    logging.info(shengbte_code)

    SinglefileData = DataFactory('singlefile')
    FORCE_CONSTANTS_2ND = SinglefileData(file=path.join(
        INPUT_DIR, 'FORCE_CONSTANTS_2ND'),
        filename='FORCE_CONSTANTS_2ND')
    FORCE_CONSTANTS_3RD = SinglefileData(file=path.join(
        INPUT_DIR, 'FORCE_CONSTANTS_3RD'),
        filename='FORCE_CONSTANTS_3RD')

    import numpy as np
    inputs = {
        'code': shengbte_code,
        'control': orm.Dict(dict={
            'allocations': {
                'nelements': 2,
                'natoms': 2,
                'ngrid': [3, 3, 3],
                'norientations': 3
            },
            'crystal': {
                'lattvec': [
                    [0, 0.5, 0.5],
                    [0.5, 0, 0.5],
                    [0.5, 0.5, 0]],
                'elements': ['In', 'As'],
                'types': [1, 2],
                'positions': [
                    [0, 0, 0],
                    [0.25, 0.25, 0.25]
                ],
                'scell': [5, 5, 5],
                'born': [
                    [
                        [2.67810, 0, 0],
                        [0, 2.67810, 0],
                        [0, 0, 2.67810]
                    ],
                    [
                        [-2.67810, 0, 0],
                        [0, -2.67810, 0],
                        [0, 0, -2.67810]
                    ]
                ],
                'orientations': [[1, 0, 0], [1, 1, 0], [1, 1, 1]]
            },
            'parameters': {
                'T': 300,
            },
            'flags': {
                'espresso': False,
                'nonanalytic': True,
                'nanowires': True
            }
        }),
        'FORCE_CONSTANTS_2ND': FORCE_CONSTANTS_2ND,
        'FORCE_CONSTANTS_3RD': FORCE_CONSTANTS_3RD,
        # 'clean_workdir': orm.Bool(True),
        'metadata': {
            'description':
            "Test job submission with the aiida_shengbte plugin",
            # 'dry_run': True,
            # 'store_provenance': False,
        },
    }

    # Note: in order to submit your calculation to the aiida daemon, do:
    # from aiida.engine import submit
    # future = submit(CalculationFactory('shengbte'), **inputs)
    result = engine.run(CalculationFactory('shengbte.shengbte'), **inputs)

    logging.info(result)

@ click.command()
@ cmdline.utils.decorators.with_dbenv()
@ cmdline.params.options.CODE()
def cli(code):
    """Run example.

    Example usage: $ ./example_01.py -- code diff@localhost

    Alternative (creates diff@localhost-test code): $ ./example_01.py

    Help: $ ./example_01.py -- help
    """
    test_run(code)

if __name__ == '__main__':
    cli()  # pylint: disable = no-value-for-parameter

max_stars_repo_name: materials-science/aiida-shengbte

max_issues_repo_name: materials-science/aiida-shengbte

max_forks_repo_name: materials-science/aiida-shengbte