text:
import torch.nn.functional as F
from collections import OrderedDict
import numpy as np

class InputPadder:
    """Pads images such that dimensions are divisible by 8 , from RAFT."""

    def __init__(self, dims, mode='sintel'):
        self.ht, self.wd = dims[-2:]
        pad_ht = (((self.ht // 8) + 1) * 8 - self.ht) % 8
        pad_wd = (((self.wd // 8) + 1) * 8 - self.wd) % 8
        if mode == 'sintel':
            self._pad = [
                pad_wd // 2, pad_wd - pad_wd // 2, pad_ht // 2,
                pad_ht - pad_ht // 2
            ]
        else:
            self._pad = [pad_wd // 2, pad_wd - pad_wd // 2, 0, pad_ht]

    def pad(self, *inputs):
        return [F.pad(x, self._pad, mode='replicate') for x in inputs]

    def unpad(self, x):
        ht, wd = x.shape[-2:]
        c = [self._pad[2], ht - self._pad[3], self._pad[0], wd - self._pad[1]]
        return x[..., c[0]:c[1], c[2]:c[3]]

def fill_order_keys(key, fill_value='_model.', fill_position=7):
    """fill order_dict keys in checkpoint, by Hao."""
    return key[0:fill_position] + fill_value + key[fill_position:]

def fix_order_keys(key, delete_value=6):
    """fix order_dict keys in checkpoint, by Hao."""
    return key[0:delete_value] + key[13:]

def fix_read_order_keys(key, start_value=7):
    """fix reading restored ckpt order_dict keys, by Hao."""
    return key[start_value:]

max_stars_repo_name: MasterHow/CSFlow

max_issues_repo_name: MasterHow/CSFlow

max_forks_repo_name: MasterHow/CSFlow


text:
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

from werkzeug.utils import secure_filename
from werkzeug.datastructures import  FileStorage

import os
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.models import Model
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Activation
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Lambda
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Input, Concatenate, UpSampling2D

from PIL import Image
import pytesseract

from flask import Flask, render_template, request,jsonify

UPLOAD_FOLDER = 'uploads/'
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

# %% [code]
image_height=1024
image_width=1024

@app.route('/')
def hello_world():
    return 'To access the webpage please visit "upload" page'

# %% [code]
def normalize(input_image):
    input_image = tf.cast(input_image, tf.float32) / 255.0
    #input_mask -= 1
    return input_image

# %% [code]
def decode_image(image):
    img=tf.io.decode_jpeg(image)
    img=tf.image.resize(img, [image_height, image_width])
    return img

# %% [code]
def decode_mask(image):
    img=tf.io.decode_jpeg(image,channels=1)
    img=tf.image.resize(img, [image_height, image_width])
    return img

# %% [code]
def process_1(file_paths):
    img = normalize(decode_image(tf.io.read_file(file_paths)))
    return img

# %% [code]
def process_2(file_paths):
    img = normalize(decode_image(tf.io.read_file(file_paths)))

    mask_path=tf.strings.regex_replace(file_paths,'.jpg','.jpeg')

    tab_mask=tf.strings.regex_replace(mask_path,"Image_Data", "Table_Data")
    col_mask=tf.strings.regex_replace(mask_path,"Image_Data", "Column_Data")

    table_mask = normalize(decode_mask(tf.io.read_file(tab_mask)))
    column_mask=normalize(decode_mask(tf.io.read_file(col_mask)))

    return img, {'table_mask':table_mask,'column_mask':column_mask}

# %% [code]
def create_mask(pred_mask1, pred_mask2):
    pred_mask1 = tf.argmax(pred_mask1, axis=-1)
    pred_mask1 = pred_mask1[..., tf.newaxis]

    pred_mask2 = tf.argmax(pred_mask2, axis=-1)
    pred_mask2 = pred_mask2[..., tf.newaxis]
    return pred_mask1[0], pred_mask2[0]

# %% [code]
def show_prediction_sample_image(dataset=None, num=1):

    model = tf.keras.models.load_model('mymodel_45')

    for image in dataset.take(num):
        pred_mask1, pred_mask2 = model.predict(image, verbose=1)
        table_mask, column_mask = create_mask(pred_mask1, pred_mask2)

        img1=tf.keras.preprocessing.image.array_to_img(image[0])
        img1.save('image.bmp')

        img2=tf.keras.preprocessing.image.array_to_img(table_mask)
        img2.save('table_mask.bmp')

        img3=tf.keras.preprocessing.image.array_to_img(column_mask)
        img3.save('column_mask.bmp')

    return 'img1','img2','img3'

# %% [code]
def generate_segment():
    img_org  = Image.open('./image.bmp')
    #img_org=img1
    img_mask = Image.open('./table_mask.bmp')
    #img_mask=img2
    img_mask = img_mask.convert('L')
    img=img_org.putalpha(img_mask)
    img_org.save('output.png')

# %% [code]
def ocr_core(filename):
    text = pytesseract.image_to_string(Image.open(filename))  # We'll use Pillow's Image class to open the image and pytesseract to detect the string in the image
    return text

# %% [code]
def get_mask(dataset=None, num=1):

    table=[]
    column=[]
    for i in dataset:
        table.append(i[1]['table_mask'])
        column.append(i[1]['column_mask'])

    model = tf.keras.models.load_model('../input/model50/all/mymodel_45')

    pred_tab=[]
    pred_col=[]
    for image, (mask1, mask2) in dataset.take(num):
        pred_mask1, pred_mask2 = model.predict(image, verbose=1)
        table_mask, column_mask = create_mask(pred_mask1, pred_mask2)
        pred_tab.append(table_mask)
        pred_col.append(column_mask)

    return table,column,pred_tab,pred_col

@app.route('/upload')
def upload_files():
   return render_template('upload.html')

@app.route('/uploader', methods = ['GET', 'POST'])
def upload_file():

   if request.method == 'POST':
      f = request.files['file']
      name=f.filename
      #location='uploads/' + name
      f.save(os.path.join(app.config['UPLOAD_FOLDER'], name))
      imgs=Image.open('uploads/'+ name)
      if(imgs.mode=='RGBA'):
          rgb_im = imgs.convert('RGB')
          fname='uploads/' + name
          rgb_im.save(fname,'jpeg')
      #f.save(secure_filename(location))
      img_path='uploads/*'
      list_ds = tf.data.Dataset.list_files(img_path)
      DATASET_SIZE = len(list(list_ds))
      test_size = DATASET_SIZE
      test = list_ds.take(test_size)
      BATCH_SIZE = 1
      BUFFER_SIZE = 1000
      test = test.map(process_1)
      test_dataset = test.batch(BATCH_SIZE)
      img1,img2,img3=show_prediction_sample_image(test_dataset)
      generate_segment()
      text=ocr_core('output.png')

      os.remove(os.path.join(app.config['UPLOAD_FOLDER'], name))

      #text=text.split('\n')
      text = text.split('\n')

      d={}
      count=1

      for i in range(0,len(text)):
          if(text[i]==""):
              k=1
          elif(text[i]== " "):
              k=2
          elif(text[i] == ''):
              k=3
          elif(text[i] ==' '):
              k=4
          else:
              d[count]=text[i]
              count=count+1

      return jsonify({'Table_data': d})

if __name__ == '__main__':
   app.run(host='0.0.0.0',debug=True, port=8080)

ext: py

max_stars_repo_name: raisulalam/tablenet

max_issues_repo_name: raisulalam/tablenet

max_forks_repo_name: raisulalam/tablenet

reason: import numpy


text:
import numpy as np
import pandas as pd
import time
import tensorflow.compat.v1 as tf
from .init_clusters import gen_init_centers, get_init_em_params
tf.disable_v2_behavior()

def run_em(X, sf, K, L=2, n_inits=5, n_init_iter=10, n_em_iter=100, \
           n_inner_iter=50, tol=1e-5, zero_inflated=True, b_overwrites=[],\
           skip_spectral=True):
    """
    run EM algorithm on the given dataframe with rows as genes and columns as cells
    """
    st = time.time()

    if L>2 and len(b_overwrites) != L-2:
        print("b_overwrites must be supplied of size L-2")
        return

    # get init params
    G, C = np.shape(X)
    X0 = (X/sf).T
    inits = gen_init_centers(X0, K, n_inits, n_init_iter, skip_spectral)
    if zero_inflated:
        em_inits = get_init_em_params(inits, G, K, L)
    else:
        em_inits = get_init_em_params(inits, G, K, L, q_base=1e-5)

    sol = em(X, sf, em_inits, K, L, n_em_iter,
             n_inner_iter, tol, zero_inflated)

    et = time.time()
    tc = et - st

    sol['total_em_time'] = tc
    sol['sf'] = sf

    return sol

def em(X, sf, inits, K, L, n_iter=100, n_inner_iter=50, tol=1e-5, zero_inflated=True):
    """
    run EM algorithm on the given init centers
    return the clustering labels with the highest log likelihood
    """

    # add prepare reduced data here

    print("start em algorithm")

    res = _em(X, sf, inits, K, L, n_iter, n_inner_iter, tol, zero_inflated)

    max_idx = np.argmax([r['llf'] for r in res])
    sol = res[max_idx]
    em_labels = np.argmax(sol['rho'], axis=1).flatten()

    sol['labels'] = em_labels

    return sol

def _em(X, sf, inits, K, L, n_iter=100, n_inner_iter=50, tol=1e-5, zero_inflated=True):

    G, C = np.shape(X)
    n_factor = np.log(C)

    x0 = X
    pilog0 = inits['pilog0']
    qlog0 = inits['qlog0']
    ss0 = sf

    def tensor_ll_rho():
        vF = a * tf.log(b/sc) - (x+a) * tf.log(1+b/sc) - \
            log_beta(a, x) - tf.log(x+a)
        t = q + tf.concat([vF, tf.broadcast_to(vL_in, [1, K, G, C])], axis=0)
        eta = tf.reduce_logsumexp(t, axis=(0), keepdims=True)
        ga = tf.reduce_sum(eta, axis=(2), keepdims=True)
        tt = pi + ga
        ll = tf.reduce_sum(tf.reduce_logsumexp(tt, axis=1))
        tt_m = tt - tf.reduce_max(tt, axis=1, keepdims=True)
        rho = tt_m - tf.reduce_logsumexp(tt_m, axis=1, keepdims=True)
        xi = rho + t - eta
        p_n = tf.exp(xi - tf.reduce_logsumexp(xi, axis=(0,1), keepdims=True))
        return ll, rho, p_n

    def log_beta(a, x):
        a_full = tf.broadcast_to(a, [L-1, K, G, C])
        x_full = tf.broadcast_to(x, [L-1, K, G, C])
        a_s = tf.reshape(a_full, (-1,))
        x_s = tf.reshape(x_full, (-1,))
        lt = tf.lbeta(tf.stack([x_s+1.0, a_s], axis=1))
        return tf.reshape(lt, (L-1, K, G, C))

    def outer_body(j, diffo, a, b, pi, q):
        def body(i, diff, at, bt):
            bt1 = at * bgt
            at1 = tf.maximum(
                at + (tf.log(bt)+agt-tf.digamma(at))/tf.polygamma(1.0, at), 1e-5)
            diff1 = tf.reduce_mean(tf.abs(at/at1-1.0)) + \
                tf.reduce_mean(tf.abs(bt/bt1-1.0))
            return [i-1, diff1, at1, bt1]

        def cond(i, diff, at, bt):
            return tf.logical_and(i > 0, diff > tol)

        vF = a * tf.log(b/sc) - (x+a) * tf.log(1+b/sc) - \
            log_beta(a, x) - tf.log(x+a)
        t = q + tf.concat([vF, tf.broadcast_to(vL_in, [1, K, G, C])], axis=0)
        eta = tf.reduce_logsumexp(t, axis=(0), keepdims=True)
        ga = tf.reduce_sum(eta, axis=(2), keepdims=True)
        tt = pi + ga
        tt_m = tt - tf.reduce_max(tt, axis=1, keepdims=True)
        rho = tt_m - tf.reduce_logsumexp(tt_m, axis=1, keepdims=True)
        pit = tf.reduce_logsumexp(rho, axis=(3), keepdims=True) - n_factor
        xi = rho + t - eta
        qt = tf.reduce_logsumexp(xi, axis=3, keepdims=True)  # sum over c
        qt1 = qt - tf.reduce_logsumexp(qt, axis=0, keepdims=True)  # sum over l
        xi_n = xi - tf.reduce_max(xi, axis=3, keepdims=True)
        p_n = tf.exp(xi_n[:L-1, :, :, :])
        pt_n = tf.reduce_sum(p_n, axis=3, keepdims=True)
        bgt = pt_n/tf.reduce_sum(p_n * (a+x)/(b+sc), axis=3, keepdims=True)
        agt = tf.reduce_sum(
            p_n * (tf.digamma(x+a)-tf.log(sc+b)), axis=3, keepdims=True) / pt_n

        i, diff_in, at1, bt1 = tf.while_loop(
            cond, body, (n_inner_iter, 1.0, a, b))
        diffo1 = tf.reduce_mean(tf.abs(a/at1-1.0)) + \
            tf.reduce_mean(tf.abs(b/bt1-1.0))
        return [j-1, diffo1, at1, bt1, pit, qt1]

    def outer_cond(j, diffo, a, b, pi, q):
        return tf.logical_and(j > 0, diffo > tol)

    a = tf.placeholder(tf.float32, shape=(L-1, K, G, 1))
    b = tf.placeholder(tf.float32, shape=(L-1, K, G, 1))
    pi = tf.placeholder(tf.float32, shape=(1, K, 1, 1))
    q = tf.placeholder(tf.float32, shape=(L, K, G, 1))
    x = tf.placeholder(tf.float32, shape=(G, C))
    vL_in = tf.placeholder(tf.float32, shape=(G, C))
    ss = tf.placeholder(tf.float32, shape=(C))

    sc = tf.broadcast_to(ss, [1, 1, 1, C])

    j, diff_out, atn, btn, pin, qn = tf.while_loop(
        outer_cond, outer_body, (n_iter, 1.0, a, b, pi, q))
    ll, rho, p_n = tensor_ll_rho()

    res = []
    for kk in range(len(inits['a0s'])):
        print("working on init number {}".format(kk))
        a0 = inits['a0s'][kk]
        b0 = inits['b0s'][kk]
        if zero_inflated:
            vL_in0 = (x0 != 0)*(-1e5)
        else:
            vL_in0 = (x0 != -1)*(-1e5)
        feed_dict = {x: x0, vL_in: vL_in0,
                     a: a0, b: b0, pi: pilog0, q: qlog0, ss: ss0}
        st = time.time()
        sess = tf.Session(config=tf.ConfigProto(log_device_placement=False))
        oo = sess.run((j, diff_out, atn, btn, pin, qn), feed_dict=feed_dict)
        ll0 = sess.run((ll, rho, p_n), feed_dict={
                       x: x0, vL_in: vL_in0, a: oo[2], b: oo[3], pi: oo[4], q: oo[5], ss: ss0})
        et = time.time()
        sess.close()

        tc = et - st
        res.append({'a': oo[2], 'b': oo[3], 'logpi': oo[4],
                    'logq': oo[5], 'rho': ll0[1], 'p_n': ll0[2], 'llf': ll0[0], 'em_time': tc})

    return res

max_stars_repo_name: wanglab-georgetown/JOINT

max_issues_repo_name: wanglab-georgetown/JOINT

max_forks_repo_name: wanglab-georgetown/JOINT


text:
import os
import argparse
import math
import numpy as np
import torch
import yaml
import logging

code_dir = os.path.abspath(__file__).replace('config.py','')
project_dir = os.path.abspath(__file__).replace('/assets/config.py','')
root_dir = project_dir.replace(project_dir.split('/')[-1],'')
model_dir = os.path.join(project_dir,'assets/')
trained_model_dir = os.path.join(project_dir,'assets/')

parser = argparse.ArgumentParser(description = 'ROMP: Monocular, One-stage, Regression of Multiple 3D People')
parser.add_argument('--tab',type = str,default = 'test',help = 'additional tabs')
parser.add_argument('--configs_yml',type = str,default = 'thirdparty/romp/assets/single_image.yml',help = 'setting for training')
parser.add_argument('--demo_image_folder',type = str,default = 'None',help = 'absolute path to the image folder containing the input images for evaluation')
parser.add_argument('-s1', '--cihp_pgn', action="store_true", help='Get body segmentation from samples')
parser.add_argument('-s2', '--octopus', action="store_true", help='Get SMPL Body model params')
parser.add_argument('-s3', '--pic2tex', action="store_true", help='Get model texture from samples')
parser.add_argument('-s4', '--densepose', action="store_true", help='Get DensePose data from samples')
parser.add_argument('-s5', '--demo_process', action="store_true", help='Get video frames pose data')
parser.add_argument('-s6', '--tex2shape', action="store_true", help='Get SMPL Body model deformation')
parser.add_argument('-s7', '--demo_run', action="store_true", help='Get final video frames')
parser.add_argument('-s8', '--render_output', action="store_true", help='Render final video frames')

mode_group = parser.add_argument_group(title='mode options')
#mode setting
mode_group.add_argument('--local_rank',type = int,default=0,help = 'local rank for distributed training')
mode_group.add_argument('--model_version',type = int,default = 1,help = 'model version')
mode_group.add_argument('--multi_person',type = bool,default = True,help = 'whether to make Multi-person Recovery')

mode_group.add_argument('--collision_aware_centermap',type = bool,default = False,help = 'whether to use collision_aware_centermap')
mode_group.add_argument('--collision_factor',type = float,default = 0.2,help = 'whether to use collision_aware_centermap')
mode_group.add_argument('--kp3d_format', type=str, default='smpl24', help='the joint defination of KP 3D joints: smpl24 or coco25')
mode_group.add_argument('--eval',type = bool,default = False,help = 'whether to evaluation')
mode_group.add_argument('--max_person',default=64,type=int,help = 'max person number')
mode_group.add_argument('--input_size',default=512,type=int,help = 'size of input image')
mode_group.add_argument('--Rot_type', type=str, default='6D', help='rotation representation type: angular, 6D')
mode_group.add_argument('--rot_dim', type=int, default=6, help='rotation representation type: 3D angular, 6D')
mode_group.add_argument('--centermap_conf_thresh',type = float,default = 0.25,help = 'whether to use centermap_conf_thresh')

model_group = parser.add_argument_group(title='model settings')
model_group.add_argument('--centermap_size', type=int, default=64, help='the size of center map')
model_group.add_argument('--deconv_num', type=int, default=0, help='the size of center map')
model_group.add_argument('--model_precision', type=str, default='fp32', help='the model precision: fp16/fp32')
#model settings
model_group.add_argument('--backbone',type = str,default = 'hrnet',help = 'backbone model: resnet50 or hrnet')
model_group.add_argument('--input-size',default = 512,type = int, help = 'input image size 512 or 256.')
model_group.add_argument('--gmodel-path',type = str,default = 'assets/ROMP_hrnet32.pkl',help = 'trained model path of generator')

train_group = parser.add_argument_group(title='training options')
#basic training setting
train_group.add_argument('--print-freq', type = int, default = 50, help = 'training epochs')
train_group.add_argument('--fine_tune',type = bool,default = True,help = 'whether to run online')
train_group.add_argument('--gpu',default='0',help='gpus',type=str)
train_group.add_argument('--batch_size',default=64,help='batch_size',type=int)
train_group.add_argument('--val_batch_size',default=1,help='valiation batch_size',type=int)
train_group.add_argument('--nw',default=4,help='number of workers',type=int)

eval_group = parser.add_argument_group(title='evaluation options')
eval_group.add_argument('--calc_PVE_error',type = bool,default =False)

dataset_group = parser.add_argument_group(title='datasets options')
#dataset setting:
dataset_group.add_argument('--dataset-rootdir',type=str, default=os.path.join(root_dir,'dataset/'), help= 'root dir of all datasets')
eval_group = parser.add_argument_group(title='evaluation options')

other_group = parser.add_argument_group(title='other options')
#visulaization settings
other_group.add_argument('--high_resolution',type = bool,default = True,help = 'whether to visulize with high resolution 500*500')

#model save path and log file
other_group.add_argument('--save-best-folder', type = str, default = os.path.join(root_dir,'checkpoints/'), help = 'Path to save models')
other_group.add_argument('--log-path', type = str, default = os.path.join(root_dir,'log/'), help = 'Path to save log file')

smpl_group = parser.add_argument_group(title='SMPL options')
#smpl info
smpl_group.add_argument('--total-param-count',type = int,default = 85, help = 'the count of param param')
smpl_group.add_argument('--smpl-mean-param-path',type = str,default = os.path.join(model_dir,'satistic_data','neutral_smpl_mean_params.h5'),
    help = 'the path for mean smpl param value')
smpl_group.add_argument('--smpl-model',type = str,default = os.path.join(model_dir,'statistic_data','neutral_smpl_with_cocoplus_reg.txt'),
    help = 'smpl model path')

smplx_group = parser.add_argument_group(title='SMPL-X options')
smpl_group.add_argument('--smplx-model',type = bool,default = True, help = 'the count of param param')
smpl_group.add_argument('--cam_dim',type = int,default = 3, help = 'the dimention of camera param')
smpl_group.add_argument('--beta_dim',type = int,default = 10, help = 'the dimention of SMPL shape param, beta')
smpl_group.add_argument('--smpl_joint_num',type = int,default = 22)
smpl_group.add_argument('--smpl_model_path',type = str,default = os.path.join(model_dir),help = 'smpl model path')
smpl_group.add_argument('--smpl_uvmap',type = str,default = os.path.join(model_dir, 'uv_table.npy'),help = 'smpl UV Map coordinates for each vertice')
smpl_group.add_argument('--smpl_female_texture',type = str,default = os.path.join(model_dir, 'SMPL_sampleTex_f.jpg'),help = 'smpl UV texture for the female')
smpl_group.add_argument('--smpl_male_texture',type = str,default = os.path.join(model_dir, 'SMPL_sampleTex_m.jpg'),help = 'smpl UV texture for the male')
smpl_group.add_argument('--smpl_J_reg_h37m_path',type = str,default = os.path.join(model_dir, 'J_regressor_h36m.npy'),help = 'SMPL regressor for 17 joints from H36M datasets')
smpl_group.add_argument('--smpl_J_reg_extra_path',type = str,default = os.path.join(model_dir, 'J_regressor_extra.npy'),help = 'SMPL regressor for 9 extra joints from different datasets')

args = parser.parse_args()
args.kernel_sizes = [5]
with open(args.configs_yml) as file:
    configs_update = yaml.full_load(file)
for key, value in configs_update['ARGS'].items():
    if isinstance(value,str):
        exec("args.{} = '{}'".format(key, value))
    else:
        exec("args.{} = {}".format(key, value))

hrnet_pretrain = os.path.join(project_dir,'assets/pretrain.pkl')
args.tab = '{}_cm{}_{}'.format(args.backbone,args.centermap_size,args.tab)


text:
import numpy as np
from sklearn.cluster import KMeans

list1 = [88.0, 74.0, 96.0,85.0]
list2 = [92.0, 99.0, 95.0,94.0]
list3 = [91.0, 87.0, 99.0,95.0]
list4 = [78.0, 99.0, 97.0,81.0]
list5 = [88.0, 78.0, 98.0,84.0]
list6 = [100.0, 95.0, 100.0,92.0]

X = np.array([list1,list2,list3,list4,list5,list6])

kmeans = KMeans(n_clusters=2).fit(X)
pred = kmeans.predict(X)
print(pred)

max_stars_repo_path: Week4/34. kmeans_SKLearn_Stu.py

max_issues_repo_path: Week4/34. kmeans_SKLearn_Stu.py

max_forks_repo_path: Week4/34. kmeans_SKLearn_Stu.py


text:
import h5py
import numpy as np
import cupy as cp
from include import symplectic as sm
from numpy import heaviside as heav
import matplotlib.pyplot as plt

"""Generates a pair of half-quantum vortex dipoles in opposite spinor components."""

def get_phase(N_vort, pos, Nx, Ny, X, Y, len_x, len_y):
    # Phase initialisation
    theta_k = np.zeros((N_vort, Nx, Ny))
    theta_tot = np.zeros((Nx, Ny))
    pi = np.pi

    for k in range(N_vort // 2):
        y_m, y_p = pos[k], pos[N_vort + k]  # y-positions
        x_m, x_p = pos[N_vort // 2 + k], pos[3 * N_vort // 2 + k]  # x-positions

        # Scaling positional arguments
        Y_minus = 2 * np.pi * (Y - y_m) / len_y
        X_minus = 2 * np.pi * (X - x_m) / len_x
        Y_plus = 2 * np.pi * (Y - y_p) / len_y
        X_plus = 2 * np.pi * (X - x_p) / len_x
        x_plus = 2 * np.pi * x_p / len_x
        x_minus = 2 * np.pi * x_m / len_x

        for nn in np.arange(-5, 5):
            theta_k[k, :, :] += np.arctan(
                np.tanh((Y_minus + 2 * pi * nn) / 2) * np.tan((X_minus - pi) / 2)) \
                                - np.arctan(np.tanh((Y_plus + 2 * pi * nn) / 2) * np.tan((X_plus - pi) / 2)) \
                                + pi * (heav(X_plus, 1.) - heav(X_minus, 1.))

        theta_k[k, :, :] -= (2 * pi * Y / len_y) * (x_plus - x_minus) / (2 * pi)
        theta_tot += theta_k[k, :, :]
    return theta_tot

# --------------------------------------------------------------------------------------------------------------------
# Spatial and Potential parameters:
# --------------------------------------------------------------------------------------------------------------------
Nx, Ny = 1024, 1024  # Number of grid pts
Mx, My = Nx // 2, Ny // 2
dx = dy = 1  # Grid spacing
dkx = np.pi / (Mx * dx)
dky = np.pi / (My * dy)  # K-space spacing
len_x = Nx * dx  # Box length
len_y = Ny * dy
x = cp.arange(-Mx, Mx) * dx
y = cp.arange(-My, My) * dy
X, Y = cp.meshgrid(x, y)  # Spatial meshgrid

kx = cp.arange(-Mx, Mx) * dkx
ky = cp.arange(-My, My) * dky
Kx, Ky = cp.meshgrid(kx, ky)  # K-space meshgrid
Kx, Ky = cp.fft.fftshift(Kx), cp.fft.fftshift(Ky)

# Controlled variables
V = 0
p = 0.
q = -0.01
c0 = 3e-5
c1 = 0.75e-5
k = 0

# Time steps, number and wavefunction save variables
Nt = 10000000
Nframe = 10000
dt = 1e-2
t = 0.

# ----------------------------------------------------------------------------------------------------------------------
# Generating initial state
# ----------------------------------------------------------------------------------------------------------------------
# Euler angles
alpha = 0.
beta = 0.
gamma = 0.

# Generate phase fields:
N_vort = 2
pos_plus = [241, -256, -241, -256]
pos_minus = [-271, -256, 271, -256]
theta_plus = get_phase(N_vort, pos_plus, Nx, Ny, cp.asnumpy(X), cp.asnumpy(Y), len_x, len_y)
theta_minus = get_phase(N_vort, pos_minus, Nx, Ny, cp.asnumpy(X), cp.asnumpy(Y), len_x, len_y)

# Initial wavefunction
n_0 = 1.6e9 / (1024 ** 2)  # Background density
Psi = cp.empty((3, Nx, Ny), dtype='complex128')
Psi[0, :, :] = cp.sqrt(n_0 / 2.) * cp.ones((Nx, Ny), dtype='complex128') * cp.exp(1j * cp.asarray(theta_plus))
Psi[1, :, :] = cp.zeros((Nx, Ny)) + 0j
Psi[2, :, :] = cp.sqrt(n_0 / 2.) * cp.ones((Nx, Ny), dtype='complex128') * cp.exp(1j * cp.asarray(theta_minus))
psi_plus, psi_0, psi_minus = sm.rotation(Psi, Nx, Ny, alpha, beta, gamma)  # Performs rotation to wavefunction

# Finding argument of each component:
theta_plus_fix = np.angle(psi_plus)
theta_minus_fix = np.angle(psi_minus)

psi_plus_k = cp.fft.fft2(psi_plus)
psi_0_k = cp.fft.fft2(psi_0)
psi_minus_k = cp.fft.fft2(psi_minus)

# Normalisation constants
N_plus = dx * dy * cp.linalg.norm(psi_plus) ** 2
N_minus = dx * dy * cp.linalg.norm(psi_minus) ** 2

# ----------------------------------------------------------------------------------------------------------------------
# Generating data file
# ----------------------------------------------------------------------------------------------------------------------
filename = 'HQV_oppComp'  # Name of file to save data to
data_path = '../data/dipole/{}.hdf5'.format(filename)

# --------------------------------------------------------------------------------------------------------------------
# Imaginary time evolution
# --------------------------------------------------------------------------------------------------------------------
for i in range(2000):
    sm.fourier_space(psi_plus_k, psi_0_k, psi_minus_k, -1j * dt, Kx, Ky, q)

    psi_plus, psi_0, psi_minus = cp.fft.ifft2(psi_plus_k), cp.fft.ifft2(psi_0_k), cp.fft.ifft2(psi_minus_k)

    F_perp, Fz, C, S, n = sm.calc_spin_dens(psi_plus, psi_0, psi_minus, -1j * dt, c1)

    psi_plus, psi_0, psi_minus = sm.interaction_flow(psi_plus, psi_0, psi_minus, C, S, Fz, F_perp, -1j * dt, V, p, c0,
                                                     n)

    psi_plus_k, psi_0_k, psi_minus_k = cp.fft.fft2(psi_plus), cp.fft.fft2(psi_0), cp.fft.fft2(psi_minus)

    sm.fourier_space(psi_plus_k, psi_0_k, psi_minus_k, -1j * dt, Kx, Ky, q)

    N_plus_new = dx * dy * cp.sum(cp.abs(cp.fft.ifft2(psi_plus_k)) ** 2)
    N_minus_new = dx * dy * cp.sum(cp.abs(cp.fft.ifft2(psi_minus_k)) ** 2)

    psi_plus_k = cp.fft.fft2(cp.sqrt(N_plus) * cp.fft.ifft2(psi_plus_k) / cp.sqrt(N_plus_new))
    psi_minus_k = cp.fft.fft2(cp.sqrt(N_minus) * cp.fft.ifft2(psi_minus_k) / cp.sqrt(N_minus_new))

    # Fixing the phase:
    psi_plus = cp.fft.ifft2(psi_plus_k)
    psi_minus = cp.fft.ifft2(psi_minus_k)
    psi_plus *= cp.exp(1j * theta_plus_fix) / cp.exp(1j * np.angle(psi_plus))
    psi_minus *= cp.exp(1j * theta_minus_fix) / cp.exp(1j * np.angle(psi_minus))
    psi_plus_k = cp.fft.fft2(psi_plus)
    psi_minus_k = cp.fft.fft2(psi_minus)

with h5py.File(data_path, 'w') as data:
    # Saving spatial data:
    data.create_dataset('grid/x', x.shape, data=cp.asnumpy(x))
    data.create_dataset('grid/y', y.shape, data=cp.asnumpy(y))

    # Saving time variables:
    data.create_dataset('time/Nt', data=Nt)
    data.create_dataset('time/dt', data=dt)
    data.create_dataset('time/Nframe', data=Nframe)

    # Creating empty wavefunction datasets to store data:
    data.create_dataset('wavefunction/psi_plus', (Nx, Ny, 1), maxshape=(Nx, Ny, None), dtype='complex64')
    data.create_dataset('wavefunction/psi_0', (Nx, Ny, 1), maxshape=(Nx, Ny, None), dtype='complex64')
    data.create_dataset('wavefunction/psi_minus', (Nx, Ny, 1), maxshape=(Nx, Ny, None), dtype='complex64')

    data.create_dataset('initial_state/psi_plus', data=cp.asnumpy(cp.fft.ifft2(psi_plus_k)))
    data.create_dataset('initial_state/psi_0', data=cp.asnumpy(cp.fft.ifft2(psi_0_k)))
    data.create_dataset('initial_state/psi_minus', data=cp.asnumpy(cp.fft.ifft2(psi_minus_k)))

# --------------------------------------------------------------------------------------------------------------------
# Real time evolution
# --------------------------------------------------------------------------------------------------------------------
for i in range(Nt):

    sm.fourier_space(psi_plus_k, psi_0_k, psi_minus_k, dt, Kx, Ky, q)

    psi_plus, psi_0, psi_minus = cp.fft.ifft2(psi_plus_k), cp.fft.ifft2(psi_0_k), cp.fft.ifft2(psi_minus_k)

    F_perp, Fz, C, S, n = sm.calc_spin_dens(psi_plus, psi_0, psi_minus, dt, c1)

    psi_plus, psi_0, psi_minus = sm.interaction_flow(psi_plus, psi_0, psi_minus, C, S, Fz, F_perp, dt, V, p, c0, n)

    psi_plus_k, psi_0_k, psi_minus_k = cp.fft.fft2(psi_plus), cp.fft.fft2(psi_0), cp.fft.fft2(psi_minus)

    sm.fourier_space(psi_plus_k, psi_0_k, psi_minus_k, dt, Kx, Ky, q)

    # Saves data
    if np.mod(i + 1, Nframe) == 0:
        # Updates file with new wavefunction values:
        with h5py.File(data_path, 'r+') as data:
            new_psi_plus = data['wavefunction/psi_plus']
            new_psi_plus.resize((Nx, Ny, k + 1))
            new_psi_plus[:, :, k] = cp.asnumpy(cp.fft.ifft2(psi_plus_k))

            new_psi_0 = data['wavefunction/psi_0']
            new_psi_0.resize((Nx, Ny, k + 1))
            new_psi_0[:, :, k] = cp.asnumpy(cp.fft.ifft2(psi_0_k))

            new_psi_minus = data['wavefunction/psi_minus']
            new_psi_minus.resize((Nx, Ny, k + 1))
            new_psi_minus[:, :, k] = cp.asnumpy(cp.fft.ifft2(psi_minus_k))

        k += 1  # Increment array index

    if np.mod(i, Nframe * 2) == 0:
        print('t = {:2f}'.format(t))

    if i == 0:
        fig, ax = plt.subplots(2, figsize=(8, 10))
        cvals = np.linspace(0, n_0 * 0.6, 25)
        ax[0].contourf(cp.asnumpy(X), cp.asnumpy(Y), cp.asnumpy(cp.abs(cp.fft.ifft2(psi_plus_k)) ** 2), cvals, cmap='gnuplot')
        ax[1].contourf(cp.asnumpy(X), cp.asnumpy(Y), cp.asnumpy(cp.abs(cp.fft.ifft2(psi_minus_k)) ** 2), cvals, cmap='gnuplot')
        ax[0].set_title(r'$|\psi_+|^2$')
        ax[1].set_title(r'$|\psi_-|^2$')
        plt.show()

    t += dt

reason: import numpy,from numpy,import cupy


text:
#!/usr/bin/env python

"""
Testing for ZeroShotNAS, aka data description driven nas
ZZ
May 14, 2020
"""

import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
import numpy as np
import os
import sys
import copy
import logging
import pickle
import pandas as pd
import argparse

from amber.modeler import EnasCnnModelBuilder
from amber.architect.controller import ZeroShotController
from amber.architect.modelSpace import State, ModelSpace
from amber.architect.commonOps import count_model_params, unpack_data

from amber.architect.trainEnv import MultiManagerEnvironment, ParallelMultiManagerEnvironment
from amber.architect.reward import LossAucReward, LossReward
from amber.plots import plot_controller_hidden_states
from amber.utils import run_from_ipython, get_available_gpus
from amber.utils.logging import setup_logger
from amber.utils.data_parser import get_data_from_simdata

from amber.modeler.modeler import build_sequential_model, KerasModelBuilder
from amber.architect.manager import GeneralManager, DistributedGeneralManager
from amber.architect.modelSpace import get_layer_shortname

from amber.utils.sequences import EncodedGenome
from amber.utils.sequences import EncodedHDF5Genome
from amber.utils.sampler import BatchedHDF5Generator

def get_controller(model_space, session, data_description_len=3, layer_embedding_sharing=None):
    with tf.device("/cpu:0"):
        controller = ZeroShotController(
            data_description_config={
                "length": data_description_len,
                #"hidden_layer": {"units":8, "activation": "relu"},
                #"regularizer": {"l1":1e-8}
                },
            share_embedding=layer_embedding_sharing,
            model_space=model_space,
            session=session,
            with_skip_connection=False,
            skip_weight=None,
            skip_target=0.2,
            lstm_size=128,
            lstm_num_layers=2,
            kl_threshold=0.05,
            train_pi_iter=100,
            optim_algo='adam',
            temperature=0.5,
            tanh_constant=2,
            buffer_type="MultiManager",
            buffer_size=5,
            batch_size=20,
            use_ppo_loss=False,
            rescale_advantage_by_reward=False
        )
    return controller

def get_model_space_common():

    # Setup and params.
    state_space = ModelSpace()
    default_params = {"kernel_initializer": "glorot_uniform",
                      "activation": "relu"}
    param_list = [
            # Block 1:
            #[
            #    {"filters": 64, "kernel_size": 8},
            #    {"filters": 64, "kernel_size": 14},
            #    {"filters": 64, "kernel_size": 20}
            #],
            # Block 2:
            #[
            #    {"filters": 128, "kernel_size": 8},
            #    {"filters": 128, "kernel_size": 14},
            #    {"filters": 128, "kernel_size": 20}
            #],
            # Block 3:
            [
                {"filters": 256, "kernel_size": 8},
                {"filters": 256, "kernel_size": 14},
                {"filters": 256, "kernel_size": 20}
            ],
        ]

    # Build state space.
    layer_embedding_sharing = {}
    conv_seen = 0
    for i in range(len(param_list)):
        # Build conv states for this layer.
        conv_states = [State("Identity")]
        for j in range(len(param_list[i])):
            d = copy.deepcopy(default_params)
            for k, v in param_list[i][j].items():
                d[k] = v
            conv_states.append(State('conv1d', name="conv{}".format(conv_seen), **d))
        state_space.add_layer(conv_seen*3, conv_states)
        if i > 0:
            layer_embedding_sharing[conv_seen*3] = 0
        conv_seen += 1

        # Add pooling states.
        if i < len(param_list) - 1:
            pool_states = [State('Identity'),
                           State('maxpool1d', pool_size=4, strides=4),
                           State('avgpool1d', pool_size=4, strides=4)]
            if i > 0:
                layer_embedding_sharing[conv_seen*3-2] = 1
        else:
            pool_states = [
                    State('Flatten'),
                    State('GlobalMaxPool1D'),
                    State('GlobalAvgPool1D')
                ]
        state_space.add_layer(conv_seen*3-2, pool_states)

        # Add dropout
        state_space.add_layer(conv_seen*3-1, [
            State('Identity'),
            State('Dropout', rate=0.1),
            State('Dropout', rate=0.3),
            State('Dropout', rate=0.5)
            ])
        if i > 0:
            layer_embedding_sharing[conv_seen*3-1] = 2

    # Add final classifier layer.
    state_space.add_layer(conv_seen*3, [
            State('Dense', units=30, activation='relu'),
            State('Dense', units=100, activation='relu'),
            State('Identity')
        ])
    return state_space, layer_embedding_sharing

def get_manager_distributed(train_data, val_data, controller, model_space, wd, data_description, verbose=0,
                            devices=None, train_data_kwargs=None, validate_data_kwargs=None, **kwargs):
    reward_fn = LossAucReward(method='auc')
    input_node = State('input', shape=(1000, 4), name="input", dtype='float32')
    output_node = State('dense', units=1, activation='sigmoid')
    model_compile_dict = {
        'loss': 'binary_crossentropy',
        'optimizer': 'adam',
        'metrics': ['acc']
    }
    mb = KerasModelBuilder(inputs=input_node, outputs=output_node, model_compile_dict=model_compile_dict, model_space=model_space)
    manager = DistributedGeneralManager(
        devices=devices,
        train_data_kwargs=train_data_kwargs,
        train_data=train_data,
        validate_data_kwargs=validate_data_kwargs,
        validation_data=val_data,
        epochs=50,
        child_batchsize=1000,
        reward_fn=reward_fn,
        model_fn=mb,
        store_fn='model_plot',
        model_compile_dict=model_compile_dict,
        working_dir=wd,
        verbose=verbose,
        save_full_model=False,
        model_space=model_space,
        fit_kwargs={
            'steps_per_epoch': 100,
            'workers': 3, 'max_queue_size': 50,
            'earlystop_patience': 5}
    )
    return manager

def get_manager_common(train_data, val_data, controller, model_space, wd, data_description, verbose=2, n_feats=1, **kwargs):
    input_node = State('input', shape=(1000, 4), name="input", dtype='float32')
    output_node = State('dense', units=n_feats, activation='sigmoid')
    model_compile_dict = {
        'loss': 'binary_crossentropy',
        'optimizer': 'adam',
        'metrics': ['acc']
    }
    session = controller.session

    reward_fn = LossAucReward(method='auc')
    gpus = get_available_gpus()
    num_gpus = len(gpus)
    mb = KerasModelBuilder(inputs=input_node, outputs=output_node, model_compile_dict=model_compile_dict, model_space=model_space, gpus=num_gpus)

    # TODO: batch_size here is not effective because it's been set at generator init
    child_batch_size = 1000*num_gpus
    manager = GeneralManager(
        train_data=train_data,
        validation_data=val_data,
        epochs=1000,
        child_batchsize=child_batch_size,
        reward_fn=reward_fn,
        model_fn=mb,
        store_fn='model_plot',
        model_compile_dict=model_compile_dict,
        working_dir=wd,
        verbose=verbose,
        save_full_model=True,
        model_space=model_space,
        fit_kwargs={
            'steps_per_epoch': 50,
            'workers': 8, 'max_queue_size': 50,
            'earlystop_patience': 20}
    )
    return manager

def get_samples_controller(dfeatures, controller, model_space, T=100):
    res = []
    for i in range(len(dfeatures)):
        dfeature = dfeatures[[i]]
        prob_arr = [ np.zeros((T, len(model_space[i]))) for i in range(len(model_space)) ]
        for t in range(T):
            a, p = controller.get_action(dfeature)
            for i, p in enumerate(p):
                prob_arr[i][t] = p.flatten()
        res.append(prob_arr)
    return res

def convert_to_dataframe(res, model_space, data_names):
    probs = []
    layer = []
    description = []
    operation = []
    for i in range(len(res)):
        for j in range(len(model_space)):
            for k in range(len(model_space[j])):
                o = get_layer_shortname(model_space[j][k])
                p = res[i][j][:, k]
                # extend the array
                T = p.shape[0]
                probs.extend(p)
                description.extend([data_names[i]]*T)
                layer.extend([j]*T)
                operation.extend([o]*T)
    df = pd.DataFrame({
        'description':description,
        'layer': layer,
        'operation': operation,
        'prob': probs
        })
    return df

def reload_trained_controller(arg):
    wd = arg.wd #wd = "./outputs/zero_shot/"
    configs, config_keys, controller, model_space = read_configs(arg)
    controller.load_weights(os.path.join(wd, "controller_weights.h5"))
    dfeatures = np.stack([configs[k]["dfeatures"] for k in config_keys])
    res = get_samples_controller(dfeatures, controller, model_space, T=1000)

    import seaborn as sns
    import matplotlib.pyplot as plt
    df = convert_to_dataframe(res, model_space, data_names=[configs[k]['feat_name'] for k in config_keys])

    for i in range(len(model_space)):
        sub_df = df.loc[ (df.layer==i) ]
        plt.clf()
        fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10,5))
        sns.boxplot(x="operation", y="prob",
            hue="description",
            data=sub_df,
            ax=ax
            )
        ax.set_xticklabels(ax.get_xticklabels(), rotation=45)
        plt.legend(bbox_to_anchor=(1.05, 1.0), loc='upper left')
        plt.tight_layout()
        plt.savefig(os.path.join(wd, "layer_%i.png"%i), bbox_inches="tight")
    df.to_csv(os.path.join(wd, "sampled_controller_arcs.tsv"), sep="\t")
    return res

def read_configs(arg):
    dfeature_names = list()
    with open(arg.dfeature_name_file, "r") as read_file:
        for line in read_file:
            line = line.strip()
            if line:
                dfeature_names.append(line)

    wd = arg.wd
    model_space, layer_embedding_sharing = get_model_space_common()
    print(layer_embedding_sharing)
    try:
        session = tf.Session()
    except AttributeError:
        session = tf.compat.v1.Session()

    controller = get_controller(
            model_space=model_space,
            session=session,
            data_description_len=len(dfeature_names),
            layer_embedding_sharing=layer_embedding_sharing)
    # Load in datasets and configurations for them.
    if arg.config_file.endswith("tsv"):
        sep = "\t"
    else:
        sep = ","
    configs = pd.read_csv(arg.config_file, sep=sep)
    tmp = list(configs.columns) # Because pandas doesn't have infer quotes...
    if any(["\"" in x for x in tmp]):
        configs = pd.read_csv(arg.config_file, sep=sep, quoting=2)
        print("Re-read with quoting")
    configs = configs.to_dict(orient='index')
    # Get available gpus for parsing to DistributedManager
    gpus = get_available_gpus()
    if len(gpus) == 0:
        gpus = [None]
    gpus_ = gpus * len(configs)

    manager_getter = get_manager_distributed if arg.parallel else get_manager_common
    config_keys = list()
    seed_generator = np.random.RandomState(seed=1337)
    for i, k in enumerate(configs.keys()):
        # Build datasets for train/test/validate splits.
        for x in ["train", "validate"]:
            if arg.lockstep_sampling is False and x == "train":
                cur_seed = seed_generator.randint(0, np.iinfo(np.uint32).max)
            else:
                cur_seed = 1337
            d = {
                        'hdf5_fp':  arg.train_file if x=='train' else arg.val_file,
                        'y_selector': configs[k]['ds_col'],
                        'batch_size': 1024 if arg.parallel else 1024*len(gpus),
                        'shuffle': x=='train',
                }
            if arg.parallel is True:
                configs[k][x] = BatchedHDF5Generator
                configs[k]['%s_data_kwargs'%x] = d
            else:
                configs[k][x] = BatchedHDF5Generator(**d)

        # Build covariates and manager.
        configs[k]["dfeatures"] = np.array(
            [configs[k][x] for x in dfeature_names]) # TODO: Make cols dynamic.

        tmp = dict() if arg.parallel is False else dict(
                train_data_kwargs=configs[k]['train_data_kwargs'],
                validate_data_kwargs=configs[k]["validate_data_kwargs"]
                )

        configs[k]["manager"] = manager_getter(
            devices=[gpus_[i]],
            train_data=configs[k]["train"],
            val_data=configs[k]["validate"],
            controller=controller,
            model_space=model_space,
            wd=os.path.join(wd, "manager_%s"%k),
            data_description=configs[k]["dfeatures"],
            dag_name="AmberDAG{}".format(k),
            verbose=0 if arg.parallel is True else 2,
            n_feats=configs[k]["n_feats"],
            **tmp
            )
        config_keys.append(k)
    return configs, config_keys, controller, model_space

def train_nas(arg):
    wd=arg.wd
    logger = setup_logger(wd, verbose_level=logging.INFO)

    gpus = get_available_gpus()
    configs, config_keys, controller, model_space = read_configs(arg)
    # Setup env kwargs.
    tmp = dict(data_descriptive_features=np.stack([configs[k]["dfeatures"] for k in config_keys]),
               controller=controller,
               manager=[configs[k]["manager"] for k in config_keys],
               logger=logger,
               max_episode=200,
               max_step_per_ep=5,
               working_dir=wd,
               time_budget="72:00:00",
               with_input_blocks=False,
               with_skip_connection=False,
               save_controller_every=1,
               resume_prev_run=arg.resume
           )

    if arg.parallel is True:
        env = ParallelMultiManagerEnvironment(
                    processes=len(gpus),
                    **tmp)
    else:
        env = MultiManagerEnvironment(**tmp)

    try:
        env.train()
    except KeyboardInterrupt:
        print("user interrupted training")
        pass
    controller.save_weights(os.path.join(wd, "controller_weights.h5"))

def get_parser():
    parser = argparse.ArgumentParser(description="experimental zero-shot nas")
    parser.add_argument("--analysis", type=str, choices=['train', 'reload'], required=True, help="analysis type")
    parser.add_argument("--wd", type=str, default="./outputs/zero_shot/", help="working dir")
    parser.add_argument("--parallel", default=False, action="store_true", help="Use parallel")
    parser.add_argument("--resume", default=False, action="store_true", help="resume previous run")
    parser.add_argument("--config-file", type=str, required=True, help="Path to the config file to use.")
    parser.add_argument("--dfeature-name-file", type=str, required=True, help="Path to file with dataset feature names listed one per line.")
    parser.add_argument("--train-file", type=str, required=True, help="Path to the hdf5 file of training data.")
    parser.add_argument("--val-file", type=str, required=True, help="Path to the hdf5 file of validation data.")
    parser.add_argument("--lockstep-sampling", default=False, action="store_true", help="Ensure same training samples used for all models.")
    return parser

if __name__ == "__main__":
    if run_from_ipython():
        arg_str = """--analysis reload     --wd  ./outputs/zero_shot_deepsea/     --config-file ./data/zero_shot_deepsea/debug_feats.config.2_cats.tsv     --train-file ./data/zero_shot_deepsea/train.h5     --val-file ./data/zero_shot_deepsea/val.h5     --dfeature-name-file ./data/zero_shot_deepsea/dfeatures_ordered_list.txt     --parallel"""
        arg_list = arg_str.split()
        sys.argv = ['foo'] + arg_list
    else:
        arg = get_parser()
        arg = parser.parse_args()

        if arg.analysis == "train":
            train_nas(arg)
        elif arg.analysis == "reload":
            reload_trained_controller(arg)
        else:
            raise Exception("Unknown analysis type: %s"% arg.analysis)


text:
import numpy as np
model = dict(
    type='RetinaNet_obb',
    pretrained='torchvision://resnet50',
    backbone=dict(
        type='ResNet',
        depth=50,
        num_stages=4,
        out_indices=(0, 1, 2, 3),
        frozen_stages=1,
        norm_cfg=dict(type='BN', requires_grad=True),
        norm_eval=True,
        style='pytorch'),
    neck=dict(
        type='FPN',
        in_channels=[256, 512, 1024, 2048],
        out_channels=256,
        start_level=1,
        add_extra_convs='on_input',
        num_outs=5),
    bbox_head=dict(
        type='RetinaHead_obb',
        num_classes=15,
        in_channels=256,
        stacked_convs=4,
        feat_channels=256,
        anchor_generator=dict(
            type='AnchorGenerator',
            octave_base_scale=4,
            scales_per_octave=3,
            ratios=[0.5, 1.0, 2.0],
            strides=[8, 16, 32, 64, 128]),
        bbox_coder=dict(
            type='DeltaXYWHBBoxCoder',
            target_means=[.0, .0, .0, .0, .0],
            target_stds=[1.0, 1.0, 1.0, 1.0, 1.0]),
        loss_cls=dict(
            type='FocalLoss',
            use_sigmoid=True,
            gamma=2.0,
            alpha=0.25,
            loss_weight=1.0),
        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
    train_cfg=dict(
        assigner=dict(
            type='MaxIoUAssigner',
            pos_iou_thr=0.5,
            neg_iou_thr=0.4,
            min_pos_iou=0,
            ignore_iof_thr=-1),
        allowed_border=-1,
        pos_weight=-1,
        debug=False),
    test_cfg=dict(
        nms_pre=2000,
        min_bbox_size=0,
        score_thr=0.05,
        nms=dict(type='nms', iou_threshold=0.5),
        max_per_img=2000))

dataset_type = 'dotaDataset'
data_root = '/content/DOTA_ssplit_600_150/'
img_norm_cfg = dict(
    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
albu_train_transforms = [
    dict(
     type='Resize',
        height=800, width=800, p=1),
    dict(
        type='RandomSizedCrop',
         min_max_height=(600, 800), height=800, width=800, p=0.5),
    # dict(
    #     type='Rotate',
    #         limit=list(np.arange(-90, 90+16, 15)),
    #       interpolation=1, border_mode=0, p=0.8),
    dict(
        type='ShiftScaleRotate',
        shift_limit=0.0625,
        scale_limit=0.2,
        rotate_limit=list(np.arange(-90, 90+16, 15)),
        interpolation=1,
        border_mode=0,
        p=.9),

    dict(
        type='OneOf',
        transforms=[
            dict(
                type='RandomBrightnessContrast',
                brightness_limit=[0.1, 0.3],
                contrast_limit=[0.1, 0.3],
                p=0.9),
            dict(
                type='HueSaturationValue',
                hue_shift_limit=.2,
                sat_shift_limit=.2,
                val_shift_limit=.2,
                p=0.9)
        ],
        p=0.2),
#     dict(type='JpegCompression', quality_lower=85, quality_upper=95, p=0.2),
#     dict(type='ChannelShuffle', p=0.1),
    dict(type='ToGray', p=0.05),
    dict(type='HorizontalFlip', p=0.5),
    dict(type='VerticalFlip', p=0.5),
    dict(
        type='OneOf',
        transforms=[
            dict(type='Blur', blur_limit=3, p=1.0),
            dict(type='MedianBlur', blur_limit=3, p=1.0)
        ],
        p=0.1),
]

train_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(type='LoadAnnotations', with_bbox=True, with_mask=True),
    dict(
        type='Albu',
        transforms=albu_train_transforms,
        bbox_params=dict(
            type='BboxParams',
            format='pascal_voc',
            label_fields=['gt_labels'],
            min_area=10,
            min_visibility=0.6,
            filter_lost_elements=True),
        keymap={
            'img': 'image',
            'gt_masks': 'masks',
            'gt_bboxes': 'bboxes'
        },
        update_pad_shape=False,
        skip_img_without_anno=True),
    dict(type='Resize', img_scale=(800, 800), keep_ratio=True),
    dict(type='Pad', size_divisor=32),
    dict(type='Normalize',
         mean=[103.53, 116.28, 123.675],
                std=[1.0, 1.0, 1.0],
                to_rgb=False),
    dict(type='DefaultFormatBundle'),
    dict(
        type='Collect',
        keys=['img', 'gt_bboxes', 'gt_labels', 'gt_masks'],
        meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg',
                   'pad_shape', 'scale_factor'))
]

test_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(
        type='MultiScaleFlipAug',
        img_scale=(800, 800),
        flip=False,
        transforms=[
            dict(type='Resize', keep_ratio=True),
            dict(type='RandomFlip'),
            dict(
                type='Normalize',
                mean=[103.53, 116.28, 123.675],
                std=[1.0, 1.0, 1.0],
                to_rgb=False),
            dict(type='Pad', size_divisor=32),
            dict(type='ImageToTensor', keys=['img']),
            dict(type='Collect', keys=['img'])
        ])
]

data = dict(
    samples_per_gpu=4,
    workers_per_gpu=2,
    train=dict(
        type='dotaDataset',
        ann_file='train/DOTA_train.json',
        img_prefix='train/images',
        pipeline=train_pipeline,
        data_root=data_root),
    val=dict(
        type='dotaDataset',
        ann_file='val/DOTA_val.json',
        img_prefix='val/images',
        pipeline=test_pipeline,
        data_root=data_root),
    test=dict(
        type='dotaDataset',
        ann_file='train/DOTA_train.json',
        img_prefix='train/images',
        pipeline=test_pipeline,
        data_root=data_root))

# evaluation = None
evaluation = dict(interval=1, metric='mAP')

# optimizer
optimizer = dict(type='SGD', lr=0.001, momentum=0.9, weight_decay=0.0001)
optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
# learning policy
lr_config = dict(
    policy='step',
    warmup='linear',
    warmup_iters=500,
    warmup_ratio=0.001,
    step=[8, 11])

runner = dict(type='EpochBasedRunner', max_epochs=12)
checkpoint_config = dict(interval=1)
log_config = dict(interval=50, hooks=[dict(type='TextLoggerHook')])
custom_hooks = [dict(type='NumClassCheckHook')]
dist_params = dict(backend='nccl')
log_level = 'INFO'
load_from = '/content/mmdetection/checkpoint/retinanet_r50_fpn_2x_coco_20200131-fdb43119.pth'
resume_from = None
workflow = [('train', 1)]
work_dir = '/content/drive/MyDrive/tutorial_exps1'
seed = 0
gpu_ids = range(0, 1)

max_stars_repo_name: nimaone/mmdetection

max_issues_repo_name: nimaone/mmdetection

max_forks_repo_name: nimaone/mmdetection


text:
# Author: Robert Luke <mail@robertluke.net>
#
# License: BSD (3-clause)

import pandas as pd
import numpy as np
import mne
from mne.transforms import apply_trans, _get_trans
from mne.utils import _validate_type
from mne.io import BaseRaw

def _read_fold_xls(fname, atlas="Juelich"):
    """Read fOLD toolbox xls file.

    The values are then manipulated in to a tidy dataframe.

    Note the xls files are not included as no license is provided.

    Parameters
    ----------
    fname : str
        Path to xls file.
    atlas : str
        Requested atlas.
    """
    page_reference = {"AAL2": 2,
                      "AICHA": 5,
                      "Brodmann": 8,
                      "Juelich": 11,
                      "Loni": 14}

    tbl = pd.read_excel(fname,
                        sheet_name=page_reference[atlas])

    # Remove the spacing between rows
    empty_rows = np.where(np.isnan(tbl["Specificity"]))[0]
    tbl = tbl.drop(empty_rows).reset_index(drop=True)

    # Empty values in the table mean its the same as above
    for row_idx in range(1, tbl.shape[0]):
        for col_idx, col in enumerate(tbl.columns):
            if not isinstance(tbl[col][row_idx], str):
                if np.isnan(tbl[col][row_idx]):
                    tbl.iloc[row_idx, col_idx] = \
                        tbl.iloc[row_idx - 1, col_idx]

    tbl["Specificity"] = tbl["Specificity"] * 100
    tbl["brainSens"] = tbl["brainSens"] * 100
    return tbl

def _generate_montage_locations(montage='standard_1005'):
    """Get standard montage locations in dataframe.

    Data is returned in the same format as the eeg_positions library.

    Parameters
    ----------
    montage : str
        Standard MNE montage to use.
    """
    montage = mne.channels.make_standard_montage(montage)
    coords = pd.DataFrame.from_dict(
        montage.get_positions()['ch_pos']).T
    coords["label"] = coords.index
    coords = coords.rename(columns={0: "x", 1: "y", 2: "z"})

    return coords.reset_index(drop=True)

def _find_closest_standard_location(position, reference, trans_pos='mri'):
    """Return closest montage label to coordinates.

    Parameters
    ----------
    position : array
        Coordinates.
    reference : dataframe
        As generated by _generate_montage_locations.
    trans_pos : str
        Apply a transformation to positions to specified frame.
        Use None for no transformation.
    """
    p0 = np.array(position)
    if trans_pos is not None:
        head_mri_t, _ = _get_trans('fsaverage', 'head', 'mri')
        p0 = apply_trans(head_mri_t, p0)

    dists = np.zeros(reference["x"].shape)

    for idx in range(len(dists)):
        p1 = np.array([reference["x"][idx],
                       reference["y"][idx],
                       reference["z"][idx]])
        dists[idx] = np.linalg.norm(p0 - p1)

    min_idx = np.argmin(dists)

    return reference["label"][min_idx]

def fold_landmark_specificity(raw, landmark, fold_files=[None],
                              atlas="Juelich"):
    """Return the specificity of each channel to a specified brain landmark.

    Specificity values as stored in the fOLD toolbox
    :footcite:`morais2018fnirs`
    excel files.

    The data is not provided with MNE-NIRS. You must download the excel
    spreadsheets from the authors website and provide the paths using
    the ``fold_files`` argument.

    Parameters
    ----------
    raw : BaseRaw
        The fNIRS data.
    landmark : str
        Landmark of interest. Must be present in fOLD toolbox data file.
    fold_files : list
        Paths to fold toolbox files.
    atlas : str
        Brain atlas to use.

    Returns
    -------
    spec : array
        Specificity values for each channel to brain landmark.

    References
    ----------
    .. footbibliography::
    """
    if None in fold_files:
        raise ValueError("You must specify the path to fOLD xls files")

    if not isinstance(landmark, str):
        raise ValueError(f"Landmark must be a string. Got {type(landmark)}")

    _validate_type(raw, BaseRaw, 'raw')

    reference_locations = _generate_montage_locations()

    fold_tbl = pd.DataFrame()
    for fname in fold_files:
        fold_tbl = fold_tbl.append(_read_fold_xls(fname, atlas=atlas))

    specificity = np.zeros(len(raw.ch_names))
    for cidx in range(len(raw.ch_names)):

        tbl = _source_detector_fold_table(raw, cidx,
                                          reference_locations, fold_tbl)

        if len(tbl) > 0:
            tbl["ContainsLmk"] = [landmark in la for la in tbl["Landmark"]]
            tbl = tbl.query("ContainsLmk == True")["Specificity"]

        if len(tbl) == 0:
            continue
            # print(f"No data for {src_name}-{det_name}")
        elif len(tbl) == 1:
            specificity[cidx] = tbl.values[0]
        else:
            raise RuntimeError("Multiple specificity values returned")

    return np.array(specificity)

def fold_channel_specificity(raw, fold_files=[None], atlas="Juelich"):
    """Return the landmarks and specificity a channel is sensitive to.

    Specificity values as stored in the fOLD toolbox
    :footcite:`morais2018fnirs`
    excel files.

    The data is not provided with MNE-NIRS. You must download the excel
    spreadsheets from the authors website and provide the paths using
    the ``fold_files`` argument.

    Parameters
    ----------
    raw : BaseRaw
        The fNIRS data.
    fold_files : list
        Paths to fold toolbox files.
    atlas : str
        Brain atlas to use.

    Returns
    -------
    spec : list of dataframes
        List of dataframes, one for each channel.

    References
    ----------
    .. footbibliography::
    """
    if None in fold_files:
        raise ValueError("You must specify the path to fOLD xls files")

    _validate_type(raw, BaseRaw, 'raw')

    reference_locations = _generate_montage_locations()

    fold_tbl = pd.DataFrame()
    for fname in fold_files:
        fold_tbl = fold_tbl.append(_read_fold_xls(fname, atlas=atlas))

    chan_spec = list()
    for cidx in range(len(raw.ch_names)):

        tbl = _source_detector_fold_table(raw, cidx,
                                          reference_locations, fold_tbl)
        chan_spec.append(tbl.reset_index(drop=True))

    return chan_spec

def _source_detector_fold_table(raw, cidx, reference_locations, fold_tbl):
    src = raw.info['chs'][cidx]['loc'][3:6]
    det = raw.info['chs'][cidx]['loc'][6:9]

    src_name = _find_closest_standard_location(src,  # noqa
                                               reference_locations)
    det_name = _find_closest_standard_location(det,  # noqa
                                               reference_locations)

    tbl = fold_tbl.query("Source == @src_name"). \
        query("Detector == @det_name")
    # Try reversing source and detector
    if len(tbl) == 0:
        tbl = fold_tbl.query("Source == @det_name"). \
            query("Detector == @src_name")

    return tbl

max_stars_repo_name: AiAiHealthcare/mne-nirs

max_issues_repo_name: AiAiHealthcare/mne-nirs

max_forks_repo_name: AiAiHealthcare/mne-nirs


text:
# Drawing a rectangle on an new canvas

# importing relevant libraries
import cv2
import numpy as np

# creating a new canvas
canvas = np.zeros((200, 200, 3), dtype="uint8")
# setting start and end coordinates of the top-left
# and bottom-right  of rectangle
start = (10, 10)
end = (100, 100)
# setting the thickness
thickness = 5

# Drawing rectangle
cv2.rectangle(canvas, start, end, color=(0, 0, 255), thickness=thickness)
# saving the modified image
cv2.imwrite("rec_photo_2.jpg", canvas)
# displaying the modified image
cv2.imshow("Rectangle", canvas)
cv2.waitKey(0)

max_stars_repo_name: sts-sadr/cvhub

max_issues_repo_name: sts-sadr/cvhub

max_forks_repo_name: sts-sadr/cvhub


text:
import numpy as np
import pandas as pd
import pptk
import matplotlib.pyplot as plt

def read_df():
    df = pd.read_csv('../data/records.csv', usecols=['X', 'Y', 'Z', 'intensity', 'raw_classification'])
    arr = df.to_numpy()
    points = arr[:, :3] / 1000
    intensities = arr[:, 3]
    labels = arr[:, 4]
    return points, intensities, labels

def fit_multiple_horizontal_planes(num_planes):
    """
    colorize the point cloud accross multiple horizontal planes
    thru it
    :param num_planes: number of planes or horizontal sections
    :return:
    """
    points_o, intensities_o, labels_o = read_df()
    heights = points_o[:, -1]
    intervals = np.linspace(int(np.min(heights)), int(np.max(heights)), num_planes)
    color_labels = np.zeros((len(points_o),))
    for i in range(len(intervals) - 1):
        mask = np.bitwise_and((intervals[i] < heights), (heights < intervals[i + 1]))
        args_filtered = np.argwhere(mask)
        color_labels[args_filtered] = i
    max_label = np.max(color_labels)
    colors = plt.get_cmap("tab20")(color_labels / (max_label if max_label > 0 else 1))[:, :-1]
    v = pptk.viewer(points_o)
    v.attributes(colors)

if __name__ == '__main__':
    fit_multiple_horizontal_planes(15)


text:
#!/usr/bin/env python

""" controller.py
Node for motor controllers.
Contains 2 controllers
1. X,Y location control
2. W rotation control
Note. The bebop will stay at a constant height
"""

import rospy
import potential_path as pp
from apriltags2_ros.msg import AprilTagDetectionArray
from apriltags2_ros.msg import AprilTagDetection
from geometry_msgs.msg import PoseWithCovarianceStamped
from std_msgs.msg import Bool
from geometry_msgs.msg import Twist
from geometry_msgs.msg import Point
from std_msgs.msg import String
import numpy as np

# An individual controller that controls a single component. One controller per algorithm.  This base class controller will output a constant speed.
class Controller(object):
    # axis is the storage place for the values returned by the controller. Initialize it with the appropriate length.
    def __init__(self, axis=(0,0,0,0,0,0)):
        self.axis = axis

    # Update method is called whenever the speed is desired.
    def update(self, location=(0,0)):
        return self.constrain(self.axis)

    # Set the speeds for the base controller
    def set_speeds(axis):
        self.axis = axis

    # Defacto constrain method to keep values within the ranges demanded by bebop_autonomy
    def constrain(self, values):
        return np.clip(values, -1,1)

# Controller for the potential_path module
class PotentialController(Controller):
    # Either provide model, goal and obstacles, or use default
    def __init__(self, model=None, goal=None, obstacles=()):
        if model is not None:
            self.model = model
        elif goal is not None:
            self.model = pp.MapPotential(goal=goal, obstacles=obstacles)
        else:
            self.model = pp.MapPotential()
        super(PotentialController, self).__init__(axis=(0,0))

    # get gradient from model and constrain it.
    def update(self, location):
        axis = self.model.gradient(location)
        return super(PotentialController, self).update(location)

    def constrain(self, values):
        print("Yay child class method called")
        #TODO write potential controller specific constraints
        np.clip(values,-1,1)

# An aggregation of controllers. In this case: Location and Rotation.
# Pushes commands out to the cmd_pub_m node at 100ms intervals
class ControllerCommand(object):
    def __init__(self):
        rospy.init_node("controller_command")
        self.rate = rospy.Rate(10) #Rate of loop in Hz
        #Topic to change Location Controller
        rospy.Subscriber("location_controller", String, callback=self.change_controller, callback_args='location')
        #Topic to change Rotation Controller
        rospy.Subscriber("rotation_controller", String, callback=self.change_controller, callback_args='rotation')
        #Topic to track April Tag
        rospy.Subscriber("tag_pixel_location", Point, self.update_tag_pixel_coordinate)
        rospy.Subscriber("tag_location", Point, self.update_tag_location)

        #Publishing Velocities
        self.cmd_pub = rospy.Publisher("bebop/cmd_vel_set", Twist, queue_size=1)
        self.cmd_vel = Twist()
        self.cmd_vel.linear.x = self.cmd_vel.linear.y = self.cmd_vel.linear.z = self.cmd_vel.angular.x = self.cmd_vel.angular.y = self.cmd_vel.angular.z = 0

        #Setting the Controllers
        self.location_controller_hover = Controller(0,0)
        self.location_controller = self.location_controller_hover
        self.rotation_controller_follow = PotentialController(
                goal=pp.GoalPotential(location=(320,184), z=0.05, kind='conic', d_threshold=50),
                obstacles=(pp.ObstaclePotential(location=(320,184),kind='circle', d_safe=50, r=180, n= 0.01, keep_out='in', max_gradient=3),))
        self.rotation_controller_search = Controller(axis=(0.25,))
        self.rotation_controller = self.rotation_controller_search
        self.tag_pixel_coordinate = (0,0)

    def change_controller(self, data, controller_type):
        if controller_type == 'location':
            if type(data) is String:
                if data.data == 'hover':
                    self.location_controller =  self.location_controller_hover
                elif data.data == 'follow':
                    pass
            #self.location_controller = data.something
            pass
        elif controller_type == 'rotation':
            if type(data) is String:
                if data.data == 'search':
                    self.rotation_controller = self.rotation_controller_search
                elif data.data == 'follow':
                    self.rotation_controller = self.rotation_controller_follow

    # Called when the apriltag location is parsed into (x,y) coordinates in pixels
    def update_tag_pixel_coordinate(self, data):
        self.tag_pixel_coordinate = (data.x, data.y)

    # Called when the apriltag location is determined relative to the odom reference frame
    def update_tag_location(self, data):
        self.tag_location = (data)

    # query controllers and publish commands at 100ms intervals
    def run(self):
        while not rospy.is_shutdown():
            self.cmd_vel.linear.x, self.cmd_vel.linear.y = self.location_controller.update(self.position)
            self.cmd_vel.angular.z = self.rotation_controller.update(self.tag_pixel_coordinate)[0]
            self.cmd_pub.publish(self.cmd_vel)
            self.rate.sleep()

if __name__=="__main__":
    cc = ControllerCommand()
    cc.run()


text:
import numpy
from sympy import Rational as frac
from sympy import sqrt

from ..helpers import book, fsd, pm
from ._ewing import ewing as stroud_cn_3_5
from ._hammer_stroud import hammer_stroud_2n as stroud_cn_5_2
from ._helpers import NCubeScheme
from ._mustard_lyness_blatt import mustard_lyness_blatt as stroud_cn_5_5
from ._phillips import phillips as stroud_cn_7_1
from ._stroud_1957 import stroud_1957_2 as stroud_cn_2_1
from ._stroud_1957 import stroud_1957_3 as stroud_cn_3_1
from ._stroud_1966 import stroud_1966_a as stroud_cn_5_4
from ._stroud_1966 import stroud_1966_b as stroud_cn_5_6
from ._stroud_1966 import stroud_1966_c as stroud_cn_5_7
from ._stroud_1966 import stroud_1966_d as stroud_cn_5_8
from ._stroud_1968 import stroud_1968 as stroud_cn_5_3
from ._thacher import thacher as stroud_cn_2_2
from ._tyler import tyler as stroud_cn_3_3

_citation = book(
    authors=["Arthur Stroud"],
    title="Approximate Calculation of Multiple Integrals",
    publisher="Prentice Hall",
    year="1971",
)

def stroud_cn_1_1(n):
    # centroid scheme
    reference_volume = 2 ** n
    weights = numpy.array([reference_volume])
    points = numpy.full((1, n), 0)
    return NCubeScheme("Stroud Cn 1-1", n, weights, points, 1, _citation)

def stroud_cn_1_2(n):
    # product trapezoidal scheme
    weights = numpy.full(2 ** n, 1)
    points = pm(n, 1)
    return NCubeScheme("Stroud Cn 1-2", n, weights, points, 1, _citation)

def stroud_cn_3_2(n):
    reference_volume = 2 ** n

    weights = numpy.full(2 * n, frac(reference_volume, 2 * n))
    r = sqrt(frac(n, 3))
    points = fsd(n, (r, 1))
    return NCubeScheme("Stroud Cn 3-2", n, weights, points, 3, _citation)

def stroud_cn_3_4(n):
    reference_volume = 2 ** n
    weights = numpy.full(2 ** n, frac(reference_volume, 2 ** n))
    r = sqrt(3) / 3
    points = pm(n, r)
    return NCubeScheme("Stroud Cn 3-4", n, weights, points, 3, _citation)

def stroud_cn_3_6(n):
    lst = n * [[frac(1, 3), frac(4, 3), frac(1, 3)]]
    weights = numpy.product(numpy.array(numpy.meshgrid(*lst)).T.reshape(-1, n), axis=-1)
    lst = n * [[-1, 0, +1]]
    points = numpy.array(numpy.meshgrid(*lst)).T.reshape(-1, n)
    return NCubeScheme("Stroud Cn 3-6", n, weights, points, 3, _citation)

def stroud_cn_5_9(n):
    lst = n * [[frac(5, 9), frac(8, 9), frac(5, 9)]]
    weights = numpy.product(numpy.array(numpy.meshgrid(*lst)).T.reshape(-1, n), axis=-1)
    sqrt35 = sqrt(frac(3, 5))
    lst = n * [[-sqrt35, 0, sqrt35]]
    points = numpy.array(numpy.meshgrid(*lst)).T.reshape(-1, n)
    return NCubeScheme("Stroud Cn 5-9", n, weights, points, 5, _citation)

__all__ = [
    "stroud_cn_1_1",
    "stroud_cn_1_2",
    "stroud_cn_2_1",
    "stroud_cn_2_2",
    "stroud_cn_3_1",
    "stroud_cn_3_2",
    "stroud_cn_3_3",
    "stroud_cn_3_4",
    "stroud_cn_3_5",
    "stroud_cn_3_6",
    # TODO implement Cn 5-1
    # Cn 5-1 is not implemented because it's based on explicit values only given for
    # n=4,5,6.
    "stroud_cn_5_2",
    "stroud_cn_5_3",
    "stroud_cn_5_4",
    "stroud_cn_5_5",
    "stroud_cn_5_6",
    "stroud_cn_5_7",
    "stroud_cn_5_8",
    "stroud_cn_5_9",
    "stroud_cn_7_1",
]

max_stars_repo_name: dariusarnold/quadpy

max_issues_repo_name: dariusarnold/quadpy

max_forks_repo_name: dariusarnold/quadpy

reason: import numpy,from sympy


text:
import argparse
import json
import warnings

import numpy as np
import pandas as pd
from pathlib import Path
from anndata import read_h5ad
from sklearn.preprocessing import MinMaxScaler
from sklearn.mixture import GaussianMixture
from vitessce import (
    VitessceConfig,
    Component as cm,
    AnnDataWrapper,
    OmeTiffWrapper,
    MultiImageWrapper,
)

# Generate binarized phenotype for a gate
def get_gate_phenotype(g, d):
    dd = d.copy()
    dd = np.where(dd < g, 0, dd)
    np.warnings.filterwarnings('ignore')
    dd = np.where(dd >= g, 1, dd)
    return dd

def get_gmm_phenotype(data):
    low = np.percentile(data, 0.01)
    high = np.percentile(data, 99.99)
    data = np.clip(data, low, high)

    sum = np.sum(data)
    median = np.median(data)
    data_med = data / sum * median

    data_log = np.log1p(data_med)
    data_log = data_log.reshape(-1, 1)

    scaler = MinMaxScaler(feature_range=(0, 1))
    data_norm = scaler.fit_transform(data_log)

    gmm = GaussianMixture(n_components=2)
    gmm.fit(data_norm)
    gate = np.mean(gmm.means_)

    return get_gate_phenotype(gate, np.ravel(data_norm))

def main(inputs, output, image, anndata, masks=None):
    """
    Parameter
    ---------
    inputs : str
        File path to galaxy tool parameter.
    output : str
        Output folder for saving web content.
    image : str
        File path to the OME Tiff image.
    anndata : str
        File path to anndata containing phenotyping info.
    masks : str
        File path to the image masks.
    """
    warnings.simplefilter('ignore')

    with open(inputs, 'r') as param_handler:
        params = json.load(param_handler)

    marker = params['marker'].strip()
    from_gate = params['from_gate']
    to_gate = params['to_gate']
    increment = params['increment']
    x_coordinate = params['x_coordinate'].strip() or 'X_centroid'
    y_coordinate = params['y_coordinate'].strip() or 'Y_centroid'

    adata = read_h5ad(anndata)

    # If no raw data is available make a copy
    if adata.raw is None:
        adata.raw = adata

    # Copy of the raw data if it exisits
    if adata.raw is not None:
        adata.X = adata.raw.X

    data = pd.DataFrame(
        adata.X,
        columns=adata.var.index,
        index=adata.obs.index
    )
    marker_values = data[[marker]].values
    marker_values_log = np.log1p(marker_values)

    # Identify the list of increments
    gate_names = []
    for num in np.arange(from_gate, to_gate, increment):
        num = round(num, 3)
        key = marker + '--' + str(num)
        adata.obs[key] = get_gate_phenotype(num, marker_values_log)
        gate_names.append(key)

    adata.obs['GMM_auto'] = get_gmm_phenotype(marker_values)
    gate_names.append('GMM_auto')

    adata.obsm['XY_coordinate'] = adata.obs[[x_coordinate, y_coordinate]].values

    vc = VitessceConfig(name=None, description=None)
    dataset = vc.add_dataset()
    image_wrappers = [OmeTiffWrapper(img_path=image, name='OMETIFF')]
    if masks:
        image_wrappers.append(
            OmeTiffWrapper(img_path=masks, name='MASKS', is_bitmask=True)
        )
    dataset.add_object(MultiImageWrapper(image_wrappers))

    dataset.add_object(
        AnnDataWrapper(
            adata,
            spatial_centroid_obsm='XY_coordinate',
            cell_set_obs=gate_names,
            cell_set_obs_names=[obj[0].upper() + obj[1:] for obj in gate_names],
            expression_matrix="X"
        )
    )
    spatial = vc.add_view(dataset, cm.SPATIAL)
    cellsets = vc.add_view(dataset, cm.CELL_SETS)
    status = vc.add_view(dataset, cm.STATUS)
    lc = vc.add_view(dataset, cm.LAYER_CONTROLLER)
    genes = vc.add_view(dataset, cm.GENES)
    cell_set_sizes = vc.add_view(dataset, cm.CELL_SET_SIZES)
    cell_set_expression = vc.add_view(dataset, cm.CELL_SET_EXPRESSION)

    vc.layout(
        (status / genes / cell_set_expression) |
        (cellsets / cell_set_sizes / lc) |
        (spatial)
    )
    config_dict = vc.export(to='files', base_url='http://localhost', out_dir=output)

    with open(Path(output).joinpath('config.json'), 'w') as f:
        json.dump(config_dict, f, indent=4)

if __name__ == '__main__':
    aparser = argparse.ArgumentParser()
    aparser.add_argument("-i", "--inputs", dest="inputs", required=True)
    aparser.add_argument("-e", "--output", dest="output", required=True)
    aparser.add_argument("-g", "--image", dest="image", required=True)
    aparser.add_argument("-a", "--anndata", dest="anndata", required=True)
    aparser.add_argument("-m", "--masks", dest="masks", required=False)

    args = aparser.parse_args()

    main(args.inputs, args.output, args.image, args.anndata, args.masks)