text:
import itertools
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
from time import time

from elastiknn.models import ElastiknnModel

from utils import open_dataset, ANNB_ROOT, Dataset, pareto_max

INDEX = "ann-benchmarks-jaccard"

def evaluate(dataset: Dataset, eknn: ElastiknnModel):
    n_neighbors = len(dataset.queries[0].indices)
    eknn.fit(dataset.corpus, shards=1)
    t0 = time()
    neighbors_pred = eknn.kneighbors([q.vector for q in dataset.queries], allow_missing=True, n_neighbors=n_neighbors)
    queries_per_sec = len(dataset.queries) / (time() - t0)
    recalls = [
        len(set(q.indices).intersection(p)) / len(q.indices)
        for (q, p) in zip(dataset.queries, neighbors_pred)
    ]
    recall = sum(recalls) / len(recalls)
    return recall, queries_per_sec

def exact(dataset: Dataset):
    eknn = ElastiknnModel(algorithm='exact', metric='jaccard', n_jobs=1, index=f"{INDEX}-exact")
    return evaluate(dataset, eknn)

def indexed(dataset: Dataset):
    eknn = ElastiknnModel(algorithm='sparse_indexed', metric='jaccard', n_jobs=1, index=f"{INDEX}-indexed")
    return evaluate(dataset, eknn)

def lsh(dataset: Dataset, bands: int = 165, rows: int = 1, candidates: float = 1.5):
    n_neighbors = len(dataset.queries[0].indices)
    eknn = ElastiknnModel(algorithm='lsh', metric='jaccard', n_jobs=1, index=f"{INDEX}-lsh",
                          mapping_params={"bands": bands, "rows": rows},
                          query_params={"candidates": int(candidates * n_neighbors)})
    return evaluate(dataset, eknn)

def main():

    dsname = "kosarak-jaccard"

    # Load the dataset. This assumes you've run the download.sh script.
    dataset = open_dataset(os.path.join(ANNB_ROOT, f"{dsname}.hdf5"))
    print(f"Loaded {len(dataset.corpus)} vectors and {len(dataset.queries)} queries")

    for _ in range(1):
        loss = exact(dataset)
        print(f"exact: {loss}")

    for _ in range(1):
        loss = indexed(dataset)
        print(f"jaccard indexed: {loss}")

    for _ in range(1):
        loss = lsh(dataset, 55, 1, 1)
        print(f"lsh: {loss}")

    bands = [('bands', b) for b in range(10, 601, 10)]
    rows = [('rows', r) for r in range(1, 2)]
    candidates = [('candidates', c) for c in np.linspace(0, 10, 21)]

    combinations = list(map(dict, itertools.product(bands, rows, candidates)))
    metrics = np.zeros((len(combinations), 2))

    for i, params in enumerate(combinations):
        print(f"Running {i + 1} of {len(combinations)}: {params}...")
        try:
            (x, y) = lsh(dataset, **params)
            print(f"Loss = {(x, y)}")
            metrics[i] = [x, y]
            pmax = pareto_max(metrics)

            plt.title(f"{dsname} results")
            plt.scatter(metrics[:, 0], metrics[:, 1], label='All')
            plt.scatter(metrics[pmax, 0], metrics[pmax, 1], label='Optimal')
            plt.legend()
            plt.savefig(f"out/{dsname}.png")
            plt.clf()

            with open(f"out/{dsname}.txt", "w") as fp:
                for j in pmax:
                    d, m = combinations[j], metrics[j]
                    fp.write(f"{d['num_bands']}, {d['num_rows']}, {m[0]}, {m[1]}\n")
        except Exception as e:
            print(e, file=sys.stderr)
            continue
        finally:
            print('-' * 100)

if __name__ == "__main__":
    main()

max_stars_repo_name: ReneHollander/elastiknn

max_issues_repo_name: ReneHollander/elastiknn

max_forks_repo_name: ReneHollander/elastiknn


text:
# Copyright 2019 The TensorTrade Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

import tensorflow as tf
import numpy as np
import pandas as pd

from gym import spaces
from typing import Dict

from tensortrade.trades import Trade, TradeType
from tensortrade.slippage import RandomSlippageModel
from tensortrade.exchanges import InstrumentExchange

class GANExchange(InstrumentExchange):
    """A simulated instrument exchange, in which the price history is based off a generative adversarial network
    model with supplied parameters.
    """

    def __init__(self, training_data: pd.DataFrame, **kwargs):
        super().__init__(data_frame=None, **kwargs)

        self._training_data = training_data
        self._prices_per_gen = kwargs.get('prices_per_gen', 1000)
        self._n_samples = kwargs.get('n_samples', 64)
        self._output_shape = kwargs.get('output_shape', (self._prices_per_gen, 5, 1))

        self._initialize_gan()

    def _initialize_gan(self):
        generator = tf.keras.Sequential([
            tf.keras.layers.InputLayer(input_shape=(1, self._n_samples)),
            tf.keras.layers.Dense(units=(self._prices_per_gen + 3) *
                                  8 * self._n_samples, activation="relu"),
            tf.keras.layers.Reshape(target_shape=((self._prices_per_gen + 3), 8, self._n_samples)),
            tf.keras.layers.Conv2DTranspose(
                filters=64, kernel_size=3, strides=(1, 1), padding="SAME", activation="relu"
            ),
            tf.keras.layers.Conv2DTranspose(
                filters=32, kernel_size=3, strides=(1, 1), padding="SAME", activation="relu"
            ),
            tf.keras.layers.Conv2DTranspose(
                filters=1, kernel_size=3, strides=(1, 1), padding="SAME", activation="sigmoid"
            ),
            tf.keras.layers.Reshape(target_shape=(self._output_shape))
        ])

        discriminator = tf.keras.Sequential([
            tf.keras.layers.InputLayer(input_shape=self._output_shape),
            tf.keras.layers.Conv2D(
                filters=32, kernel_size=3, strides=(2, 2), activation="relu"
            ),
            tf.keras.layers.Conv2D(
                filters=64, kernel_size=3, strides=(2, 2), activation="relu"
            ),
            tf.keras.layers.Flatten(),
            tf.keras.layers.Dense(units=1, activation="sigmoid"),
        ])

        self._gan = {'generator': generator,
                     'discriminator': discriminator}

    def reset(self):
        super().reset()

ext: py

max_stars_repo_name: andreskull/tensortrade

max_issues_repo_name: plutusmens/tensortrade

max_forks_repo_name: plutusmens/tensortrade

reason: import numpy


text:
# coding: utf-8
from __future__ import print_function
from __future__ import division

import torch
from torch.autograd import Variable
import numpy as np
import pickle
from collections import deque, Counter
import pdb

class RnnParameterData(object):
    def __init__(self, loc_emb_size=500, uid_emb_size=40, voc_emb_size=50, tim_emb_size=10, hidden_size=500,
                 lr=1e-3, lr_step=3, lr_decay=0.1, dropout_p=0.5, L2=1e-5, clip=5.0, optim='Adam',
                 history_mode='avg', attn_type='dot', epoch_max=30, rnn_type='LSTM', model_mode="simple",
                 data_path='/home/local/ASUAD/ychen404/Code/DeepMove_new/data/', save_path='../results/',
                data_name='foursquare_nyc_20000', accuracy_mode='top1'):
        self.data_path = data_path
        self.save_path = save_path
        self.data_name = data_name

        data = pickle.load(open(self.data_path + self.data_name + '.pk', 'rb'))
        self.vid_list = data['vid_list']
        self.uid_list = data['uid_list']
        self.data_neural = data['data_neural']

        self.tim_size = 48
        self.loc_size = len(self.vid_list)
        self.uid_size = len(self.uid_list)
        self.loc_emb_size = loc_emb_size
        self.tim_emb_size = tim_emb_size
        self.voc_emb_size = voc_emb_size
        self.uid_emb_size = uid_emb_size
        self.hidden_size = hidden_size

        self.epoch = epoch_max
        self.dropout_p = dropout_p
        self.use_cuda = True
        self.lr = lr
        self.lr_step = lr_step
        self.lr_decay = lr_decay
        self.optim = optim
        self.L2 = L2
        self.clip = clip

        self.attn_type = attn_type
        self.rnn_type = rnn_type
        self.history_mode = history_mode
        self.model_mode = model_mode
        self.accuracy_mode = accuracy_mode

def generate_input_history(data_neural, mode, mode2=None, candidate=None):
    data_train = {}
    train_idx = {}
    if candidate is None:
        candidate = data_neural.keys()
    for u in candidate:
        sessions = data_neural[u]['sessions']
        train_id = data_neural[u][mode]
        data_train[u] = {}
        for c, i in enumerate(train_id):
            if mode == 'train' and c == 0:
                continue
            session = sessions[i]
            trace = {}
            loc_np = np.reshape(np.array([s[0] for s in session[:-1]]), (len(session[:-1]), 1))
            tim_np = np.reshape(np.array([s[1] for s in session[:-1]]), (len(session[:-1]), 1))
            # voc_np = np.reshape(np.array([s[2] for s in session[:-1]]), (len(session[:-1]), 27))
            target = np.array([s[0] for s in session[1:]])
            trace['loc'] = Variable(torch.LongTensor(loc_np))
            trace['target'] = Variable(torch.LongTensor(target))
            trace['tim'] = Variable(torch.LongTensor(tim_np))
            # trace['voc'] = Variable(torch.LongTensor(voc_np))

            history = []
            if mode == 'test':
                test_id = data_neural[u]['train']
                for tt in test_id:
                    history.extend([(s[0], s[1]) for s in sessions[tt]])
            for j in range(c):
                history.extend([(s[0], s[1]) for s in sessions[train_id[j]]])
            history = sorted(history, key=lambda x: x[1], reverse=False)

            # merge traces with same time stamp
            if mode2 == 'max':
                history_tmp = {}
                for tr in history:
                    if tr[1] not in history_tmp:
                        history_tmp[tr[1]] = [tr[0]]
                    else:
                        history_tmp[tr[1]].append(tr[0])
                history_filter = []
                for t in history_tmp:
                    if len(history_tmp[t]) == 1:
                        history_filter.append((history_tmp[t][0], t))
                    else:
                        tmp = Counter(history_tmp[t]).most_common()
                        if tmp[0][1] > 1:
                            history_filter.append((history_tmp[t][0], t))
                        else:
                            ti = np.random.randint(len(tmp))
                            history_filter.append((tmp[ti][0], t))
                history = history_filter
                history = sorted(history, key=lambda x: x[1], reverse=False)
            elif mode2 == 'avg':
                history_tim = [t[1] for t in history]
                history_count = [1]
                # pdb.set_trace()
                last_t = history_tim[0]
                count = 1
                for t in history_tim[1:]:
                    if t == last_t:
                        count += 1
                    else:
                        history_count[-1] = count
                        history_count.append(1)
                        last_t = t
                        count = 1
            ################

            history_loc = np.reshape(np.array([s[0] for s in history]), (len(history), 1))
            history_tim = np.reshape(np.array([s[1] for s in history]), (len(history), 1))
            trace['history_loc'] = Variable(torch.LongTensor(history_loc))
            trace['history_tim'] = Variable(torch.LongTensor(history_tim))
            if mode2 == 'avg':
                trace['history_count'] = history_count

            data_train[u][i] = trace
        train_idx[u] = train_id
    return data_train, train_idx

def generate_input_long_history2(data_neural, mode, candidate=None):
    data_train = {}
    train_idx = {}
    if candidate is None:
        candidate = data_neural.keys()
    for u in candidate:
        sessions = data_neural[u]['sessions']
        train_id = data_neural[u][mode]
        data_train[u] = {}

        trace = {}
        session = []
        for c, i in enumerate(train_id):
            session.extend(sessions[i])
        target = np.array([s[0] for s in session[1:]])

        loc_tim = []
        loc_tim.extend([(s[0], s[1]) for s in session[:-1]])
        loc_np = np.reshape(np.array([s[0] for s in loc_tim]), (len(loc_tim), 1))
        tim_np = np.reshape(np.array([s[1] for s in loc_tim]), (len(loc_tim), 1))
        trace['loc'] = Variable(torch.LongTensor(loc_np))
        trace['tim'] = Variable(torch.LongTensor(tim_np))
        trace['target'] = Variable(torch.LongTensor(target))
        data_train[u][i] = trace
        # train_idx[u] = train_id
        if mode == 'train':
            train_idx[u] = [0, i]
        else:
            train_idx[u] = [i]
    return data_train, train_idx

def generate_input_long_history(data_neural, mode, candidate=None):
    data_train = {}
    train_idx = {}
    if candidate is None:
        candidate = data_neural.keys()
    for u in candidate:
        sessions = data_neural[u]['sessions']
        train_id = data_neural[u][mode]
        # pdb.set_trace()
        data_train[u] = {}
        for c, i in enumerate(train_id):
            trace = {}
            if mode == 'train' and c == 0:
                continue
            session = sessions[i]
            target = np.array([s[0] for s in session[1:]])

            history = []
            if mode == 'test':
                test_id = data_neural[u]['train']
                for tt in test_id:
                    history.extend([(s[0], s[1]) for s in sessions[tt]])
            for j in range(c):
                history.extend([(s[0], s[1]) for s in sessions[train_id[j]]])

            history_tim = [t[1] for t in history]
            history_count = [1]
            last_t = history_tim[0]
            count = 1
            for t in history_tim[1:]:
                if t == last_t:
                    count += 1
                else:
                    history_count[-1] = count
                    history_count.append(1)
                    last_t = t
                    count = 1

            history_loc = np.reshape(np.array([s[0] for s in history]), (len(history), 1))
            history_tim = np.reshape(np.array([s[1] for s in history]), (len(history), 1))
            trace['history_loc'] = Variable(torch.LongTensor(history_loc))
            trace['history_tim'] = Variable(torch.LongTensor(history_tim))
            trace['history_count'] = history_count

            loc_tim = history
            loc_tim.extend([(s[0], s[1]) for s in session[:-1]])
            loc_np = np.reshape(np.array([s[0] for s in loc_tim]), (len(loc_tim), 1))
            tim_np = np.reshape(np.array([s[1] for s in loc_tim]), (len(loc_tim), 1))
            trace['loc'] = Variable(torch.LongTensor(loc_np))
            trace['tim'] = Variable(torch.LongTensor(tim_np))
            trace['target'] = Variable(torch.LongTensor(target))
            data_train[u][i] = trace
        train_idx[u] = train_id
    return data_train, train_idx

def generate_queue(train_idx, mode, mode2):
    """return a deque. You must use it by train_queue.popleft()"""
    user = train_idx.keys()
    train_queue = deque()
    if mode == 'random':
        initial_queue = {}
        for u in user:
            if mode2 == 'train':
                initial_queue[u] = deque(train_idx[u][1:])
            else:
                initial_queue[u] = deque(train_idx[u])
        queue_left = 1
        while queue_left > 0:
            np.random.shuffle(user)
            for j, u in enumerate(user):
                if len(initial_queue[u]) > 0:
                    train_queue.append((u, initial_queue[u].popleft()))
                if j >= int(0.01 * len(user)):
                    break
            queue_left = sum([1 for x in initial_queue if len(initial_queue[x]) > 0])
    elif mode == 'normal':
        for u in user:
            for i in train_idx[u]:
                train_queue.append((u, i))
    return train_queue

def get_acc(target, scores):
    """target and scores are torch cuda Variable"""
    target = target.data.cpu().numpy()

    val, idxx = scores.data.topk(10, 1)
    predx = idxx.cpu().numpy()

    acc = np.zeros((3, 1))
    for i, p in enumerate(predx):
        # pdb.set_trace()
        t = target[i]
        if t in p[:10] and t > 0:
            acc[0] += 1
        if t in p[:5] and t > 0:
            acc[1] += 1
        if t == p[0] and t > 0:
            acc[2] += 1
    return acc

def run_rnn(data, run_idx, mode, accuracy_mode, lr, clip, model, optimizer, criterion, mode2=None):
    """mode=train: return model, avg_loss
       mode=test: return avg_loss,avg_acc,users_rnn_acc"""
    run_queue = None
    if mode == 'train':
        model.train(True)
        run_queue = generate_queue(run_idx, 'random', 'train')
    elif mode == 'test':
        model.train(False)
        run_queue = generate_queue(run_idx, 'normal', 'test')
    total_loss = []
    queue_len = len(run_queue)

    users_acc = {}
    for c in range(queue_len):
        optimizer.zero_grad()
        u, i = run_queue.popleft()
        if u not in users_acc:
            users_acc[u] = [0, 0]
        loc = data[u][i]['loc'].cuda()
        tim = data[u][i]['tim'].cuda()
        target = data[u][i]['target'].cuda()
        uid = Variable(torch.LongTensor([u])).cuda()

        # if mode2 in ['simple', 'simple_long']:
        scores = model(loc, tim)

        if scores.data.size()[0] > target.data.size()[0]:
            scores = scores[-target.data.size()[0]:]
        loss = criterion(scores, target)

        if mode == 'train':
            loss.backward()
            # gradient clipping
            try:
                torch.nn.utils.clip_grad_norm(model.parameters(), clip)
                for p in model.parameters():
                    if p.requires_grad:
                        p.data.add_(-lr, p.grad.data)
            except:
                pass
            optimizer.step()
        elif mode == 'test':
            users_acc[u][0] += len(target)
            acc = get_acc(target, scores)
            if accuracy_mode == 'top5':
                users_acc[u][1] += acc[1]
            elif accuracy_mode == 'top1':
                users_acc[u][1] += acc[2]

        # print(loss.data.cpu().numpy().size())
        # print(loss.item())
        # print(loss.data.cpu().numpy())
        # total_loss.append(loss.data.cpu().numpy()[0])
        # fixed indices problem
        total_loss.append(loss.data.cpu().numpy())

    avg_loss = np.mean(total_loss, dtype=np.float64)
    if mode == 'train':
        return model, avg_loss
    elif mode == 'test':
        users_rnn_acc = {}
        for u in users_acc:
            tmp_acc = users_acc[u][1] / users_acc[u][0]
            users_rnn_acc[u] = tmp_acc.tolist()[0]
        avg_acc = np.mean([users_rnn_acc[x] for x in users_rnn_acc])
        return avg_loss, avg_acc, users_rnn_acc


text:
## python
import time
import os
import subprocess
import cPickle
import sys
import math
import numpy
import random
## spider
from appionlib import spyder
## appion
from appionlib import apImage
from appionlib import apEMAN
from appionlib import apParam
from appionlib import apDisplay
from appionlib import apFile
from appionlib.apSpider import operations
from pyami import peakfinder, spider, correlator

"""
A large collection of SPIDER functions for BACKPROJECTION purposes only

I try to keep the trend
image file:
	*****img.spi
image stack file:
	*****stack.spi
doc/keep/reject file:
	*****doc.spi
file with some data:
	*****data.spi

that way its easy to tell what type of file it is
"""

#===============================
def backprojectCG(stackfile, eulerdocfile, volfile, numpart, pixrad, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	stackfile = spyder.fileFilter(stackfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(stackfile+dataext):
		apDisplay.printError("stack file not found: "+stackfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
	apFile.removeFile(volfile+dataext)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("BP CG",
		stackfile+"@*****", #stack file
		"1-%d"%(numpart), #number of particles
		str(pixrad), #particle radius
		eulerdocfile, #angle doc file
		"N", #has symmetry?, does not work
		volfile, #filename for volume
 		"%.1e,%.1f" % (1.0e-5, 0.0), #error, chi^2 limits
 		"%d,%d" % (25,1), #iterations, 1st derivative mode
 		"2000", #lambda - higher=less sensitive to noise
	)
	mySpider.close()
	apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return

#===============================
def backprojectRP(stackfile, eulerdocfile, volfile, pixrad, classnum, lambDa, numpart=None, selfile=None, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	stackfile = spyder.fileFilter(stackfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	if selfile is not None:
		selfile = spyder.fileFilter(selfile)
	volfile = spyder.fileFilter(volfile)
	counter = spyder.fileFilter("numiter")
	if not os.path.isfile(stackfile+dataext):
		apDisplay.printError("stack file not found: "+stackfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
	apFile.removeFile(volfile+dataext)

	if (numpart is not None):
		selection = "1-"+str(numpart)
	elif (selfile is not None):
		selection = selfile
	else:
		apDisplay.printError("Partilce selection is invalid for BP RP. Please make sure either numpart or selfile is specified in the function backprojectRP!")
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("BP RP x11",
		stackfile+"@*****", #stack file
		selection, #selection file or particle range
		str(pixrad), #particle radius
		eulerdocfile, #angle doc file
		"*", #has symmetry?, does not work
		volfile, #filename for volume
 		"%.1e,%.1f" % (lambDa, 0.0), #lambda, correction
 		"%d,%d" % (50,1), #iteration limit, mode
 		"%d,%d" % (0,0), #minimum, maximum
 		"%.5f" % ((1/(1+6*lambDa))*0.999), #smoothing constant = (1/(1+6*lambda))*0.999
	)
	mySpider.toSpider("SD 1,x11", str(classnum)+"/"+str(counter))
	mySpider.close()
	apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return

#===============================
def backproject3F(stackfile, eulerdocfile, volfile, numpart, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	stackfile = spyder.fileFilter(stackfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(stackfile+dataext):
		apDisplay.printError("stack file not found: "+stackfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)
	apFile.removeFile(volfile+dataext)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("BP 3F",
		stackfile+"@*****", #stack file
		"1-%d"%(numpart), #number of particles
		eulerdocfile, #angle doc file
		"*", #input symmetry file, '*' for skip
		volfile, #filename for volume
	)
	mySpider.close()
	apDisplay.printColor("finished backprojection in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return

#===============================
def projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext=".spi"):
	"""
	project 3D volumes using given Euler angles
	"""
	starttime = time.time()

	volfile = spyder.fileFilter(volfile)
	eulerdocfile = spyder.fileFilter(eulerdocfile)
	projstackfile = spyder.fileFilter(projstackfile)
	if not os.path.isfile(volfile+dataext):
		apDisplay.printError("volume file not found: "+volfile+dataext)
	if not os.path.isfile(eulerdocfile+dataext):
		apDisplay.printError("euler doc file not found: "+eulerdocfile+dataext)

	apFile.removeFile(projstackfile)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)
	mySpider.toSpider("PJ 3Q",
		volfile, #input vol file
		str(pixrad), #pixel radius
		"1-%d"%(numpart), #number of particles
		eulerdocfile, #Euler DOC file
		projstackfile+"@*****", #output projections
	)
	mySpider.close()
	apDisplay.printColor("finished projections in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return

#===============================
def crossCorrelateAndShift(infile, reffile, alignfile, ccdocfile, numpart, dataext=".spi"):
	### rewriten to do the whole thing in memory in SPIDER, it should be faster
	starttime = time.time()
	infile = spyder.fileFilter(infile)
	reffile = spyder.fileFilter(reffile)
	alignfile = spyder.fileFilter(alignfile)
	partimg = "_4"
	ccmap = "_5"
	windccmap = "_6"

	boxsize = apFile.getBoxSize(infile+dataext)

	if not os.path.isfile(infile+dataext):
		apDisplay.printError("input stack file not found: "+infile+dataext)
	if not os.path.isfile(reffile+dataext):
		apDisplay.printError("reference stack file not found: "+reffile+dataext)
	nproc = apParam.getNumProcessors()
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, nproc=nproc, log=False)

	### Allocate empty stack
	mySpider.toSpiderQuiet(
		"MS I", #command
		"_2@", #name
		"%d,%d,%d"%(boxsize), #boxsize
		str(numpart+1), #num part to create in memory
		str(numpart+1), #max particle number
	)

	partnum = 0
	while partnum < numpart:
		partnum+=1

		mySpider.toSpiderQuiet("CP",
			infile+("@%05d"%(partnum)), #picture
			partimg,
		)

		### cross correlate images; reversed order to avoid -1*shift

		mySpider.toSpiderQuiet("CC N",
			reffile+("@%05d"%(partnum)), #reference
			partimg, #picture
			ccmap, #output file
		)

		### cannot shift more the 1/4 size of the image
		mySpider.toSpiderQuiet("FI x52", partimg, "12" )
		mySpider.toSpiderQuiet("x54=int(x52/2)") #window size
		mySpider.toSpiderQuiet("x55=int(x52/4)") #window topleft
		mySpider.toSpiderQuiet("WI",
			ccmap, #input file
			windccmap, #output file
			"x54,x54", #window size
			"x55,x55", #window origin
		)

		### find the cross-correlation peak
		mySpider.toSpiderQuiet("x56=int(x52/4)+1") #center of window
		mySpider.toSpiderQuiet("PK M x11,x12,x13,x14",
			windccmap, #input ccmap file
			"x56,x56", #origin coordinates
		)

		### save info to doc file
		mySpider.toSpiderQuiet("SD %d,x13,x14"%(partnum),
			ccdocfile, #input ccmap file
		)

		### shift the images images
		mySpider.toSpiderQuiet("SH",
			partimg, #old stack
			("_2@%05d"%(partnum)), #new stack
			"x13,x14", #shift value file
		)
	### finish up
	#save stack to file
	mySpider.toSpiderQuiet(
		"CP", "_2@",
		alignfile+"@",
	)
	#delete stack
	mySpider.toSpiderQuiet(
		"DE", "_2",
	)
	mySpider.close()

	apDisplay.printColor("finished shifting particles in "+apDisplay.timeString(time.time()-starttime), "cyan")

	return

#===============================
def rctParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi"):
	"""
	inputs:
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	starttime = time.time()
	### create corresponding projections
	projstackfile = "projstack%s-%03d.spi"%(timestamp, iternum)
	projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext)

	### clean up files
	ccdocfile = "ccdocfile%s-%03d.spi"%(timestamp, iternum)
	apFile.removeFile(ccdocfile)
	alignstackfile = "alignstack%s-%03d.spi"%(timestamp, iternum)
	apFile.removeFile(alignstackfile)

	### align particles to projection
	apDisplay.printMsg("Shifting particles")
	crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart)

	if not os.path.isfile(alignstackfile):
		apDisplay.printError("aligned stack file not found: "+alignstackfile)
	apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return alignstackfile

###########################################
#														#
#	OTR functions									#
#														#
###########################################
#===============================
def otrParticleShift(volfile, origstackfile, eulerdocfile, iternum, numpart, pixrad, timestamp, dataext=".spi", classnum=None):
	"""
	inputs:
		volume
		stack, in spider format
		eulerdocfile
	outputs:
		volume
	"""
	### create corresponding projections
	if classnum is not None:
		projstackfile = str(classnum)+"/"+"projstack-%03d.spi"%(iternum)
	else:
		projstackfile = "projstack-%03d.spi"%(iternum)
	projectVolume(volfile, eulerdocfile, projstackfile, numpart, pixrad, dataext)

	### clean up files
	if classnum is not None:
		ccdocfile = str(classnum)+"/"+"ccdocfile-%03d.spi"%(iternum)
	else:
		ccdocfile = "ccdocfile-%03d.spi"%(iternum)
	apFile.removeFile(ccdocfile)

	if classnum is not None:
		alignstackfile = str(classnum)+"/"+"alignstack-%03d.spi"%(iternum)
	else:
		alignstackfile = "alignstack-%03d.spi"%(iternum)
	apFile.removeFile(alignstackfile)

	### align particles to projection
	starttime = time.time()
	apDisplay.printMsg("Shifting particles")
	crossCorrelateAndShift(origstackfile, projstackfile, alignstackfile, ccdocfile, numpart)

	if not os.path.isfile(alignstackfile):
		apDisplay.printError("aligned stack file not found: "+alignstackfile)
	apDisplay.printColor("finished correlations in "+apDisplay.timeString(time.time()-starttime), "cyan")
	return alignstackfile

#===============================
def alignAPSH(volfile, origstackfile, eulerdocfile, classnum, boxsize, numpart, pixrad, timestamp, iternum, dataext=".spi"):

	apshAngularFile = str(classnum)+"/"+"apshAngularFile-%03d.spi"%(iternum)
	if os.path.isfile(apshAngularFile):
		os.remove(apshAngularFile)
		apDisplay.printColor("File exist! Removing file "+apshAngularFile, "cyan")
	apshAngularFile = spyder.fileFilter(apshAngularFile)

	apshListFile = str(classnum)+"/"+"apshListFile-%03d.spi"%(iternum)
	if os.path.isfile(apshListFile):
		os.remove(apshListFile)
		apDisplay.printColor("File exist! Removing file "+apshListFile, "cyan")
	apshListFile = spyder.fileFilter(apshListFile)

	apshOutFile = str(classnum)+"/"+"apshOut-%03d.spi"%(iternum)
	if os.path.isfile(apshOutFile):
		os.remove(apshOutFile)
		apDisplay.printColor("File exist! Removing file "+apshOutFile, "cyan")
	apshOut = spyder.fileFilter(apshOutFile)

	apshProjStack = str(classnum)+"/"+"apshProjStack-%03d.spi"%(iternum)
	if os.path.isfile(apshProjStack):
		os.remove(apshProjStack)
		apDisplay.printColor("File exist! Removing file "+apshProjStack, "cyan")
	apshProjStack = spyder.fileFilter(apshProjStack)

	origstackfile = spyder.fileFilter(origstackfile)
	volfile = spyder.fileFilter(volfile)
	origeulerdocfile = spyder.fileFilter(eulerdocfile)

	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
	mySpider.toSpider("VO EA,x53",
		"10", #angular increment
		"0,90.0", #Range of Theta
		"0,359.9", #Range of Phi
		apshAngularFile, #Angular file name
	)
	mySpider.toSpider("DOC CREATE",
		apshListFile, #List file name
		"1",
		"1-x53"
	)
	#mySpider.toSpider("MS",
	#	"_9@", #reference projections
	#	str(boxsize),str(boxsize),"1", #boxsize
	#	"x53",
	#)
	mySpider.toSpider("PJ 3Q",
		volfile, #input vol file
		str(pixrad), #pixel radius
		apshListFile, #number of particles
		apshAngularFile, #Euler DOC file
		apshProjStack+"@*****", #output projections
	)
	mySpider.toSpider("AP SH",
		apshProjStack+"@*****", #reference projections
		"1-x53", #reference numbers
		"6,2", #translational search range, step size
		"1,%d"%(boxsize/3), #first and last ring
		apshAngularFile,
		origstackfile+"@*****",
		"1-%d"%(numpart),
		origeulerdocfile,
		"20", #"(0.0)", # no restrictions for alignment
		"1", # mirror check?
		apshOut, # output
	)

	mySpider.close()
	return apshOutFile

#===============================
def centerVolume(volfile, outvolfile, dataext=".spi"):
	volfile = spyder.fileFilter(volfile)
	outvolfile = spyder.fileFilter(outvolfile)
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
	mySpider.toSpider("CG PH x11,x12,x13,x21,x22,x23",
		volfile, #volume
	)
	mySpider.toSpider("SH F",
		volfile, #volume
		outvolfile, #output volume
		"-x21,-x22,-x23", #shifts in x y z
	)
	mySpider.close()
	return

#===============================
def calcFSC(volfile1, volfile2, fscout, dataext=".spi"):
	volfile1 = spyder.fileFilter(volfile1)
	volfile2 = spyder.fileFilter(volfile2)
	fscout = spyder.fileFilter(fscout)
	mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
	mySpider.toSpider("RF 3",
		volfile1, #volume 1
		volfile2, #volume 2
		"0.5", # ring width for RF 3
		"0.5,1.5", # scale factor - lower
		#"1.5", # scale factor - upper
		"C", # use 'C' for missing cone and 'W' for missing wedge
		"90", # maximum tilt angle in data for OTR
		"3", # factor for noise comparison ("sigma")
		fscout, # fsc curve output
	)
	mySpider.close()
	return

#===============================
def rotshiftParticle(origstackfile, partnum, rotation, Xshift, Yshift, mirror, iternum, timestamp, classnum=None, dataext=".spi"):

	origstack = spyder.fileFilter(origstackfile)

	if classnum is not None:
		apshstackfile = str(classnum)+"/"+"apshStack-%03d.spi"%(iternum)
	else:
		apshstackfile = "apshStack-%03d.spi"%(iternum)

	apshstack = spyder.fileFilter(apshstackfile)

	mySpider = spyder.SpiderSession(dataext=dataext, logo=False, log=False)
	mySpider.toSpiderQuiet("RT SQ",
		origstack+"@"+str(partnum), #stack and particle number
		apshstack+"@"+str(partnum), #output stack and particle
		"("+str(rotation)+")", #rotation
		"("+str(Xshift)+","+str(Yshift)+")", #shift parameters
	)

	if mirror==-1:
		mySpider.toSpiderQuiet("MR",
			apshstack+"@"+str(partnum), #stack and particle number
			apshstack+"@"+str(partnum), #output stack and particle
			"Y",
		)

	mySpider.close()

	return apshstackfile

#=============================== NOT WORKING CORRECTLY
def rotshiftStack(origstackfile, rotShiftFile, timestamp, iternum, classnum=None, dataext=".spi"):

	origstackfile = spyder.fileFilter(origstackfile)

	if classnum is not None:
		apshstackfile = str(classnum)+"/"+"apshstack%s.spi"%(timestamp)
	else:
		apshstackfile = "apshstack%s.spi"%(timestamp)

	apshstackfile = spyder.fileFilter(apshstackfile)

	mySpider = spyder.SpiderSession(dataext=dataext, logo=True, log=False)
	mySpider.toSpider("RT SQ",
		origstackfile+"@*****", #stack and particle number
		apshstackfile+"@*****", #output stack and particle
		"("+str(rotation)+")", #rotation
		"("+str(Xshift)+","+str(Yshift)+")", #shift parameters
	)
	mySpider.close()
	return

###########################################
#														#
#	Spider filtering functions					#
#														#
###########################################

#===============================
def butterworthLP(volfile, pixelsize, dataext=".spi"):
	"""
	inputs:
		volume
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(volfile+dataext):
		apDisplay.printError("volume file not found: "+volfile+dataext)

	mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
	mySpider.toSpider("FQ",
		volfile, #filename for volume
		"_1",
#		volfile+"_filtered", #filename for output volume
		"(7)",
		"(%.5f,%.5f)" % (pixelsize/25,pixelsize/15), #pass-band and stop-band
	)
	mySpider.close()
	apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan")
	return

#===============================
def butterworthFscLP(volfile, fscout, dataext=".spi"):
	"""
	inputs:
		volume
		fsc output file
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(volfile+dataext):
		apDisplay.printError("volume file not found: "+volfile+dataext)

	if not os.path.isfile(fscout):
		apDisplay.printError("fsc output file not found: "+fscout+dataext)

	fscfile = open(fscout, "r")
	prevValue = None
	for line in fscfile.readlines():
		value = line.split()
		try:
			int(value[0])
		except:
			#apDisplay.printMsg(line)
			continue

		if float(value[4]) < 0.5:
			if prevValue is None:
				passband = float(value[2])
				stopband = float(value[2])+0.1
			else:
				passband = prevValue
				stopband = prevValue + 0.1
			break
		prevValue = float(value[2])

	fscfile.close()

	mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
	mySpider.toSpider("FQ",
		volfile, #filename for volume
		volfile+"_filtered", #filename for output volume
		"(7)",
		"(%.5f,%.5f)" % (passband, stopband), #pass-band and stop-band
	)
	mySpider.close()
	apDisplay.printColor("finished filtering the volume "+apDisplay.timeString(time.time()-starttime), "cyan")
	return

#===============================
def normalizeVol(volfile, dataext=".spi"):
	"""
	inputs:
		volume
	outputs:
		volume
	"""
	### setup
	starttime = time.time()
	volfile = spyder.fileFilter(volfile)
	if not os.path.isfile(volfile+dataext):
		apDisplay.printError("volume file not found: "+volfile+dataext)

	mySpider = spyder.SpiderSession(dataext=dataext, logo=True)
	### read out the statistics of the volume
	mySpider.toSpider("FS x11,x12",
		volfile, #filename for volume
	)
	mySpider.toSpider("IF(x12.LT.0.0)x12=-x12")
	### set all values to positive
	mySpider.toSpider("AR",
		volfile, #filename for volume
		"_1",
		"(P1+x12)",
	)
	### save file
	mySpider.toSpider("CP",
		"_1",
		volfile, #filename for volume
	)

	mySpider.close()
	apDisplay.printColor("finished normalizing the volume to set all values to be positive"+apDisplay.timeString(time.time()-starttime), "cyan")
	return

max_issues_repo_name: vosslab/ctfeval

max_forks_repo_name: vossman/ctfeval


text:
from absl import flags, app
from src import lexutils
import json
import numpy as np
import random
import sys

FLAGS = flags.FLAGS

flags.DEFINE_string("lexfile", default=None,
                    help='Seed lexicon path')

flags.DEFINE_string("trainfile", default=None,
                    help='qa and code files')

flags.DEFINE_string("testfile", default=None,
                    help='qa and code files')

def read_file(path, seperator=' ||| '):
    inputs, outputs = [], []
    with open(path) as f:
        for d in f:
            input, output, *_ = d.split(seperator)
            inputs.append(input.strip().split(' '))
            outputs.append(output.strip().split(' '))
    return (inputs, outputs)

MASK = "UNK"

def masked_fill_(array, mask, value):
    for i, m in enumerate(mask):
        if m:
            array[i] = value

def swap_ids(tensor, id1, id2, substitute=False):
    if substitute:
        masked_fill_(tensor, tensor == id1, id2)
    else:
        masked_fill_(tensor, tensor == id1, MASK)
        masked_fill_(tensor, tensor == id2, id1)
        masked_fill_(tensor, tensor == MASK, id2)

def make_a_swap_single(inp, out, lex_and_swaps, steps=0, substitute=False):
    lexicon, swapables = lex_and_swaps['lexicon'], lex_and_swaps['swapables']

    keys = list(filter(lambda k: k in inp, lexicon.keys()))

    if len(keys) != 0:
        k1 = random.choice(keys)
        weights = [1 / max(iter(lexicon[k].values())) for k in swapables[k1]]
        k2 = random.choices(swapables[k1], weights=weights, k=1)[0]
        ks = [k1, k2]
    else:
        return

    ks_q_id = ks
    swap_ids(inp, *ks_q_id, substitute=substitute)

    if substitute:
        for v, _ in lexicon[ks[0]].items():
            code2 = random.choice(list(lexicon[ks[1]].keys()))
            masked_fill_(out, out == v, code2)
    else:
        for v, _ in lexicon[ks[0]].items():
            masked_fill_(out, out == v, MASK)

        for v, _ in lexicon[ks[1]].items():
            code1 = random.choice(list(lexicon[ks[0]].keys()))
            masked_fill_(out, out == v, code1)

        code2 = random.choice(list(lexicon[ks[1]].keys()))

        masked_fill_(out, out == MASK, code2)

def main(_):
    lex = json.load(open(FLAGS.lexfile))
    # for k, v in lex["swapables"].items():
    #     lex["swapables"][k] = np.unique(v).tolist()

    # for k in list(lex["swapables"].keys()):
    #     if len(lex["swapables"][k]) == 0:
    #         del lex["swapables"][k]
    #         if k in lex["lexicon"]:
    #             del lex["lexicon"][k]
    #     else:
    #         if k not in lex["lexicon"]:
    #             del lex["swapables"][k]
    #         for ki, vi in lex["swapables"].items():
    #             try:
    #                 idx = vi.index(k)
    #                 del vi[idx]
    #             except:
    #                 pass
    # pdb.set_trace()
    # print(lex["lexicon"])
    inputs, outputs = read_file(FLAGS.trainfile, seperator=' ||| ')
    test_inputs, test_outputs = read_file(FLAGS.testfile, seperator='\t')
    test_inputs = [" ".join(x) for x in test_inputs]
    test_outputs = [" ".join(x) for x in test_outputs]

    train_outputs = set([" ".join(x) for x in outputs])
    train_inputs = set([" ".join(x) for x in inputs])
    test_inputs_set = set(test_inputs)
    #test_compounds = set(list(open("/afs/csail.mit.edu/u/a/akyurek/akyurek/git/align/CoGnition/data/cg-test/cg-test.compound").readlines()))

    generated = set([])
    for i in range(10):
        for inp, out in zip(inputs, outputs):
            aug_inp = np.array(inp.copy(), dtype=object)
            aug_out = np.array(out.copy(),  dtype=object)
            make_a_swap_single(aug_inp, aug_out, lex)
            aug_inp = " ".join(aug_inp)
            aug_out = " ".join(aug_out)
            inp = " ".join(inp)
            out = " ".join(out)
            if aug_inp not in generated and aug_out not in train_outputs and aug_inp not in train_inputs:
                generated.add(aug_inp)
                print(aug_inp, "\t", aug_out, '\t', inp, '\t', out)
            if len(generated) > 2500:
                break
        if len(generated) > 2500:
            break

if __name__ == "__main__":
    app.run(main)

max_stars_repo_name: ekinakyurek/lexsym

max_issues_repo_name: ekinakyurek/lexsym

max_forks_repo_name: ekinakyurek/lexsym


text:
from datetime import datetime
import numpy as np
import tensorflow as tf

SEED = 69429

def get_seed():
    return SEED

def create_csv(results):
    """
    Create a csv file from the results dictionary output by create_submission_dict function

    :param results: submission dict
    """
    csv_fname = 'results_'
    csv_fname += datetime.now().strftime('%b%d_%H-%M-%S') + '.csv'

    with open(csv_fname, 'w') as f:
        f.write('ImageId,EncodedPixels,Width,Height\n')

        for key, value in results.items():
            f.write(key + ',' + str(value) + ',' + '256' + ',' + '256' + '\n')

def rle_encode(predicted_mask):
    """
    Given a predicted image mask with 0 and 1, it returns the different values to be used to create the csv

    :param predicted_mask: a predicted mask of an image
    :return string containing the value that represents all bounding box for the submission
    """
    # Flatten column-wise
    pixels = predicted_mask.T.flatten()
    pixels = np.concatenate([[0], pixels, [0]])
    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
    runs[1::2] -= runs[::2]
    return ' '.join(str(x) for x in runs)

def create_submission_dict(image_ids: list, predictions: list):
    """
    Given a list of image id and its prediction list, it returns the submission dict used in create_csv function

    :param image_ids: list of image id
    :param predictions: list of prediction masks
    :return submission dict
    """
    results = {}

    for i in range(len(predictions)):
        image_id = image_ids[i]
        predicted_mask = tf.cast(predictions[i] > 0.5, tf.float32)

        key = image_id
        value = rle_encode(np.array(predicted_mask))
        results[key] = value
    return results

def get_image_ids(filename_list: list):
    """
    Given a list of filenames, returns the list of image id

    :param filename_list: list of filename of images
    :return list of image id
    """
    image_ids = np.apply_along_axis(lambda x: x[0].replace("img/", "").replace(".tif", ""), axis=1,
                                    arr=np.reshape(filename_list, (len(filename_list), 1)))
    return image_ids


text:
from itertools import islice
from sympy import primorial, sieve

def numerator(n):
    return sum(primorial(n) // p for p in islice(sieve, n))

def sequence(n):
    return numerator(n) - primorial(n)

reason: from sympy


text:
# -*- coding: UTF-8 -*-
#! /usr/bin/python

from matplotlib import pyplot as plt
import numpy as np

# ... creates miller equilibrium with default parameters
from caid.cad_geometry import miller_equilibrium
geo = miller_equilibrium(n=[31,31], p=[3,3])

geo.plotMesh(MeshResolution=3)
plt.savefig("plasma_equilibrium_2d_miller_ex1.png")
# ...

plt.clf()

# ... creates miller equilibrium
rmin=0.3 ; rmax=1.

params_shape = {}
params_shape['A']         = 3.17
params_shape['psi_tilde'] = 0.77
params_shape['kappa0']    = 1.66
params_shape['delta0']    = 0.416
params_shape['alpha']     = 1.22

params_eq = {}
params_eq['sk']  = 0.7
params_eq['sd']  = 1.37
params_eq['dR0'] =-0.354
params_eq['q']   = 3.03
params_eq['s']   = 2.47

from caid.cad_geometry import miller_equilibrium
geo = miller_equilibrium(rmin=rmin, rmax=rmax, n=[31,31], p=[3,3],
                         params_shape=params_shape, params_eq=params_eq)
geo.plotMesh(MeshResolution=3)
plt.savefig("plasma_equilibrium_2d_miller_ex2.png")
# ...

plt.clf()

# ... creates miller equilibrium with default parameters
from caid.cad_geometry import miller_equilibrium
geo = miller_equilibrium(rmin=0.4,n=[31,31], p=[3,3])

nrb = geo[0]
points = nrb.points
points[0,:,0] = points[0,:,0].mean()
points[0,:,1] = points[0,:,1].mean()
nrb.set_points(points)
geo.plotMesh(MeshResolution=3)
plt.savefig("plasma_equilibrium_2d_miller_ex3.png")
# ...

plt.clf()

# ... creates miller equilibrium
from caid.cad_geometry import miller_equilibrium
geo = miller_equilibrium(rmin=0.4, n=[31,31], p=[3,3])

#     print boundaries info in order to know what is the internal boundary
geo[0].plotBoundariesInfo()
plt.savefig("plasma_equilibrium_2d_miller_ex4_info.png")
plt.clf()

#     now that we know that the internal face is 1
#     we can use it to split the geometry into 5 patchs
geo = geo.to5patchs(1)
geo.plotMesh(MeshResolution=3)
plt.savefig("plasma_equilibrium_2d_miller_ex4.png")
# ...

max_stars_repo_name: ratnania/caid

max_issues_repo_name: pyccel/caid

max_forks_repo_name: pyccel/caid


text:
import nengo
import nengo.spa as spa
import numpy as np

D = 256

D_space = 256
vocab_space = spa.Vocabulary(D_space)

model = spa.SPA()
with model:

    model.rule = spa.State(D)

    model.objs = spa.State(D_space, feedback=1, vocab=vocab_space)

    model.status = spa.State(D)

    actions = spa.Actions(
        'dot(rule, OBJ1*S+BELOW*V+OBJ2*O)*0.4 -dot(rule, (ABOVE+LEFT+RIGHT)*V) +'
        '(dot(objs, OBJ1*Y) - dot(objs, OBJ2*Y))+0.3 --> '
                 'status=BAD, '
                 'objs=-0.1*Y*OBJ1 + 0.1*Y*OBJ2',
        'dot(rule, OBJ1*S+BELOW*V+OBJ2*O)*0.4 - '
        '(dot(objs, OBJ1*Y) - dot(objs, OBJ2*Y))-0.3 --> '
                  'status=GOOD',

        'dot(rule, OBJ1*S+ABOVE*V+OBJ2*O)*0.4 -dot(rule, (BELOW+LEFT+RIGHT)*V) +'
        '(dot(objs, OBJ2*Y) - dot(objs, OBJ1*Y))+0.3 --> '
                 'status=BAD, '
                 'objs=-0.1*Y*OBJ2 + 0.1*Y*OBJ1',
        'dot(rule, OBJ1*S+ABOVE*V+OBJ2*O)*0.4 - '
        '(dot(objs, OBJ2*Y) - dot(objs, OBJ1*Y))-0.3 --> '
                  'status=GOOD',

        'dot(rule, OBJ1*S+LEFT*V+OBJ2*O)*0.4-dot(rule,(ABOVE+RIGHT+BELOW)*V) +'
        '(dot(objs, OBJ1*X) - dot(objs, OBJ2*X))+0.3 --> '
                 'status=BAD, '
                 'objs=-0.1*X*OBJ1 + 0.1*X*OBJ2',
        'dot(rule, OBJ1*S+LEFT*V+OBJ2*O)*0.4 - '
        '(dot(objs, OBJ1*X) - dot(objs, OBJ2*X))-0.3 --> '
                  'status=GOOD',

        'dot(rule, OBJ1*S+RIGHT*V+OBJ2*O)*0.4-dot(rule,(ABOVE+BELOW+LEFT)*V) +'
        '(dot(objs, OBJ2*X) - dot(objs, OBJ1*X))+0.3 --> '
                 'status=BAD, '
                 'objs=-0.1*X*OBJ2 + 0.1*X*OBJ1',
        'dot(rule, OBJ1*S+RIGHT*V+OBJ2*O)*0.4 - '
        '(dot(objs, OBJ2*X) - dot(objs, OBJ1*X))-0.3 --> '
                  'status=GOOD',

    )

    def input(t):
        t = t % 0.8
        if t< 0.4:
            return 'OBJ1*S+ABOVE*V+OBJ2*O'
        #elif t<0.8:
        #    return 'OBJ1*S+LEFT*V+OBJ2*O'
        else:
            return 'OBJ1*S+LEFT*V+OBJ2*O'

    model.input = spa.Input(rule=input)

    model.bg = spa.BasalGanglia(actions)

    model.thal = spa.Thalamus(model.bg)

    def display_node(t, x):
        return x

    display = nengo.Node(display_node, size_in=4)

    nengo.Connection(model.objs.output, display[0],
              transform=[vocab_space.parse('OBJ1*X').v])
    nengo.Connection(model.objs.output, display[1],
              transform=[vocab_space.parse('OBJ1*Y').v])
    nengo.Connection(model.objs.output, display[2],
              transform=[vocab_space.parse('OBJ2*X').v])
    nengo.Connection(model.objs.output, display[3],
              transform=[vocab_space.parse('OBJ2*Y').v])


text:
#%% 0.0 Imports
import pandas as pd
import inflection
# import numpy as np

#%% 0.1 Helper Functions

#%% 0.1 Loading data

df_sales_raw = pd.read_csv('data/train.csv', low_memory = False)
df_store_raw = pd.read_csv('data/store.csv', low_memory = False)
df_raw = pd.merge(df_sales_raw, df_store_raw, how='left', on='Store')
# print(df_raw.columns)

#%% 1.0 Data Description

cols_old = ['Store', 'DayOfWeek', 'Date', 'Sales', 'Customers', 'Open', 'Promo',
            'StateHoliday', 'SchoolHoliday', 'StoreType', 'Assortment',
            'CompetitionDistance', 'CompetitionOpenSinceMonth',
            'CompetitionOpenSinceYear', 'Promo2', 'Promo2SinceWeek',
            'Promo2SinceYear', 'PromoInterval']
snakecase = lambda x: inflection.underscore(x)
cols_new = list(map(snakecase, cols_old))
df_raw.columns = cols_new

# #%% 1.2 Data Dimension

# print('Number of Rows:{}'.format(df_raw.shape[0]))
# print('Number of Cols:{}'.format(df_raw.shape[1]))

# #%% 1.2 Data Types

# df_raw['date'] = pd.to_datetime(df_raw['date'])
# print(df_raw.dtypes)

# #%% 1.2 Check NA
# print(df_raw.isna().sum())

# #%%

max_stars_repo_name: rodrigolmiranda/MurdochLab

max_issues_repo_name: rodrigolmiranda/MurdochLab

max_forks_repo_name: rodrigolmiranda/MurdochLab


text:
import os
import numpy as np
import galsim
import pytest

from ..lsst_bits import get_flagval
from ..saturation import BAND_SAT_VALS
from ..masking import add_bright_star_mask
from ..artifacts.star_bleeds import add_bleed
from ..stars import StarCatalog
from ..galaxies import make_galaxy_catalog
from ..psfs import make_fixed_psf
from ..sim import make_sim

@pytest.mark.skipif(
    "CATSIM_DIR" not in os.environ,
    reason='simulation input data is not present')
@pytest.mark.parametrize('band', ('r', 'i', 'z'))
def test_star_mask_and_bleed(band):
    dims = (100, 100)

    cen = [50, 50]

    image = np.zeros(dims)
    bmask = np.zeros(dims, dtype='i4')
    pos = galsim.PositionD(x=cen[1], y=cen[0])
    mag = 12
    band = 'i'

    add_bleed(
        image=image,
        bmask=bmask,
        pos=pos,
        mag=mag,
        band=band,
    )
    add_bright_star_mask(
        bmask=bmask,
        x=pos.x,
        y=pos.y,
        radius=10,
        val=get_flagval('BRIGHT'),
    )

    assert bmask[cen[0], cen[1]] == get_flagval('SAT') | get_flagval('BRIGHT')
    assert image[cen[0], cen[1]] == BAND_SAT_VALS[band]

@pytest.mark.skipif(
    "CATSIM_DIR" not in os.environ,
    reason='simulation input data is not present')
def test_star_mask_in_sim():
    """
    test star masking using the keyword to the sim
    """
    rng = np.random.RandomState(234)

    bands = ['r']
    coadd_dim = 100
    buff = 0
    star_density = 100
    psf = make_fixed_psf(psf_type='moffat')

    for i in range(1000):
        galaxy_catalog = make_galaxy_catalog(
            rng=rng,
            gal_type='exp',
            coadd_dim=coadd_dim,
            buff=buff,
            layout='random',
        )
        star_catalog = StarCatalog(
            rng=rng,
            coadd_dim=coadd_dim,
            buff=buff,
            density=star_density,
        )
        sim_data = make_sim(
            rng=rng,
            galaxy_catalog=galaxy_catalog,
            star_catalog=star_catalog,
            coadd_dim=coadd_dim,
            bands=bands,
            psf=psf,
            g1=0, g2=0,
            star_bleeds=True,
        )

        exp = sim_data['band_data'][bands[0]][0]

        # import lsst.afw.display as afw_display
        # display = afw_display.getDisplay(backend='ds9')
        # display.mtv(exp)
        # display.scale('log', 'minmax')

        mask = exp.mask.array
        image = exp.image.array

        wsat = np.where((mask & get_flagval('SAT')) != 0)
        wbright = np.where(mask & get_flagval('BRIGHT') != 0)
        if (wsat[0].size > 0 and
                np.all(image[wsat] == BAND_SAT_VALS['r']) and
                wbright[0].size > 0):

            some_were_saturated = True
            break

    assert some_were_saturated

@pytest.mark.skipif(
    "CATSIM_DIR" not in os.environ,
    reason='simulation input data is not present')
def test_star_mask_in_sim_repeatable():
    """
    test star masking using the keyword to the sim
    """

    seed = 234

    bands = ['r']
    coadd_dim = 100
    buff = 0
    star_density = 100
    psf = make_fixed_psf(psf_type='moffat')

    for step in (0, 1):
        rng = np.random.RandomState(seed)
        for i in range(1000):
            galaxy_catalog = make_galaxy_catalog(
                rng=rng,
                gal_type='exp',
                coadd_dim=coadd_dim,
                buff=buff,
                layout='random',
            )
            star_catalog = StarCatalog(
                rng=rng,
                coadd_dim=coadd_dim,
                buff=buff,
                density=star_density,
            )
            sim_data = make_sim(
                rng=rng,
                galaxy_catalog=galaxy_catalog,
                star_catalog=star_catalog,
                coadd_dim=coadd_dim,
                bands=bands,
                psf=psf,
                g1=0, g2=0,
                star_bleeds=True,
            )

            exp = sim_data['band_data'][bands[0]][0]

            mask = exp.mask.array
            wsat = np.where((mask & get_flagval('SAT')) != 0)

            if wsat[0].size > 0:

                if step == 0:
                    nmarked = wsat[0].size
                    break
                else:
                    assert wsat[0].size == nmarked
                    break

max_stars_repo_name: aguinot/descwl-shear-sims

max_issues_repo_name: aguinot/descwl-shear-sims

max_forks_repo_name: aguinot/descwl-shear-sims


text:
import sys
import copy
from textwrap import dedent

import numpy as np
import pandas as pd

from ..client import inject_client
from ..neuroncriteria import NeuronCriteria, neuroncriteria_args
from ..synapsecriteria import SynapseCriteria
from ..utils import tqdm, iter_batches
from .connectivity import fetch_adjacencies

@inject_client
@neuroncriteria_args('neuron_criteria')
def fetch_synapses(neuron_criteria, synapse_criteria=None, batch_size=10, *, client=None):
    """
    Fetch synapses from a neuron or selection of neurons.

    Args:

        neuron_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`):
            Determines which bodies to fetch synapses for.

            Note:
                Any ROI criteria specified in this argument does not affect
                which synapses are returned, only which bodies are inspected.

        synapse_criteria (SynapseCriteria):
            Optional. Allows you to filter synapses by roi, type, confidence.
            See :py:class:`.SynapseCriteria` for details.

            If the criteria specifies ``primary_only=True`` only primary ROIs will be returned in the results.
            If a synapse does not intersect any primary ROI, it will be listed with an roi of ``None``.
            (Since 'primary' ROIs do not overlap, each synapse will be listed only once.)
            Otherwise, all ROI names will be included in the results.
            In that case, some synapses will be listed multiple times -- once per intersecting ROI.
            If a synapse does not intersect any ROI, it will be listed with an roi of ``None``.

        batch_size:
            To improve performance and avoid timeouts, the synapses for multiple bodies
            will be fetched in batches, where each batch corresponds to N bodies.
            This argument sets the batch size N.

        client:
            If not provided, the global default :py:class:`.Client` will be used.

    Returns:

        DataFrame in which each row represent a single synapse.
        Unless ``primary_only`` was specified, some synapses may be listed more than once,
        if they reside in more than one overlapping ROI.

    Example:

        .. code-block:: ipython

            In [1]: from neuprint import NeuronCriteria as NC, SynapseCriteria as SC, fetch_synapses
               ...: fetch_synapses(NC(type='ADL.*', regex=True, rois=['FB']),
               ...:                SC(rois=['LH(R)', 'SIP(R)'], primary_only=True))
            Out[1]:
                    bodyId  type     roi      x      y      z  confidence
            0   5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            1   5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            2   5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            3   5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            4   5812983094   pre   LH(R)   5447  21281  19155    0.991000
            5   5812983094   pre   LH(R)   5434  21270  19201    0.995000
            6   5812983094   pre   LH(R)   5434  21270  19201    0.995000
            7   5812983094   pre   LH(R)   5434  21270  19201    0.995000
            8   5812983094   pre   LH(R)   5447  21281  19155    0.991000
            9   5812983094   pre   LH(R)   5447  21281  19155    0.991000
            10  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            11  5812983094   pre   LH(R)   5434  21270  19201    0.995000
            12  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            13  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            14  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            15  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            16  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            17  5812983094   pre   LH(R)   5434  21270  19201    0.995000
            18  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            19  5812983094   pre   LH(R)   5434  21270  19201    0.995000
            20  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            21  5812983094   pre   LH(R)   5434  21270  19201    0.995000
            22  5812983094   pre   LH(R)   5447  21281  19155    0.991000
            23  5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            24  5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            25  5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            26  5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            27  5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            28  5812983094   pre  SIP(R)  15300  25268  14043    0.992000
            29   764377593  post   LH(R)   8043  21587  15146    0.804000
            30   764377593  post   LH(R)   8057  21493  15140    0.906482
            31   859152522   pre  SIP(R)  13275  25499  13629    0.997000
            32   859152522   pre  SIP(R)  13275  25499  13629    0.997000
            33   859152522  post  SIP(R)  13349  25337  13653    0.818386
            34   859152522  post  SIP(R)  12793  26362  14202    0.926918
            35   859152522   pre  SIP(R)  13275  25499  13629    0.997000

    """
    neuron_criteria.matchvar = 'n'
    q = f"""
        {neuron_criteria.global_with(prefix=8)}
        MATCH (n:{neuron_criteria.label})
        {neuron_criteria.all_conditions(prefix=8)}
        RETURN n.bodyId as bodyId
    """
    bodies = client.fetch_custom(q)['bodyId'].values

    batch_dfs = []
    for batch_bodies in tqdm(iter_batches(bodies, batch_size)):
        batch_criteria = copy.copy(neuron_criteria)
        batch_criteria.bodyId = batch_bodies
        batch_df = _fetch_synapses(batch_criteria, synapse_criteria, client)
        batch_dfs.append( batch_df )

    if batch_dfs:
        return pd.concat( batch_dfs, ignore_index=True )

    # Return empty results, but with correct dtypes
    dtypes = {'bodyId': np.dtype('int64'),
              'type': pd.CategoricalDtype(categories=['pre', 'post'], ordered=False),
              'roi': np.dtype('O'),
              'x': np.dtype('int32'),
              'y': np.dtype('int32'),
              'z': np.dtype('int32'),
              'confidence': np.dtype('float32')}

    return pd.DataFrame([], columns=dtypes.keys()).astype(dtypes)

def _fetch_synapses(neuron_criteria, synapse_criteria, client):
    neuron_criteria.matchvar = 'n'

    if synapse_criteria is None:
        synapse_criteria = SynapseCriteria()

    if synapse_criteria.primary_only:
        return_rois = {*client.primary_rois}
    else:
        return_rois = {*client.all_rois}

    # If the user specified rois to filter synapses by, but hasn't specified rois
    # in the NeuronCriteria, add them to the NeuronCriteria to speed up the query.
    if synapse_criteria.rois and not neuron_criteria.rois:
        neuron_criteria.rois = {*synapse_criteria.rois}
        neuron_criteria.roi_req = 'any'

    # Fetch results
    cypher = dedent(f"""\
        {neuron_criteria.global_with(prefix=8)}
        MATCH (n:{neuron_criteria.label})
        {neuron_criteria.all_conditions('n', prefix=8)}

        MATCH (n)-[:Contains]->(ss:SynapseSet),
              (ss)-[:Contains]->(s:Synapse)

        {synapse_criteria.condition('n', 's', prefix=8)}
        // De-duplicate 's' because 'pre' synapses can appear in more than one SynapseSet
        WITH DISTINCT n, s

        RETURN n.bodyId as bodyId,
               s.type as type,
               s.confidence as confidence,
               s.location.x as x,
               s.location.y as y,
               s.location.z as z,
               apoc.map.removeKeys(s, ['location', 'confidence', 'type']) as syn_info
    """)
    data = client.fetch_custom(cypher, format='json')['data']

    # Assemble DataFrame
    syn_table = []
    for body, syn_type, conf, x, y, z, syn_info in data:
        # Exclude non-primary ROIs if necessary
        syn_rois = return_rois & {*syn_info.keys()}
        # Fixme: Filter for the user's ROIs (drop duplicates)
        for roi in syn_rois:
            syn_table.append((body, syn_type, roi, x, y, z, conf))

        if not syn_rois:
            syn_table.append((body, syn_type, None, x, y, z, conf))

    syn_df = pd.DataFrame(syn_table, columns=['bodyId', 'type', 'roi', 'x', 'y', 'z', 'confidence'])

    # Save RAM with smaller dtypes and interned strings
    syn_df['type'] = pd.Categorical(syn_df['type'], ['pre', 'post'])
    syn_df['roi'] = syn_df['roi'].apply(lambda s: sys.intern(s) if s else s)
    syn_df['x'] = syn_df['x'].astype(np.int32)
    syn_df['y'] = syn_df['y'].astype(np.int32)
    syn_df['z'] = syn_df['z'].astype(np.int32)
    syn_df['confidence'] = syn_df['confidence'].astype(np.float32)

    return syn_df

@inject_client
@neuroncriteria_args('source_criteria', 'target_criteria')
def fetch_synapse_connections(source_criteria=None, target_criteria=None, synapse_criteria=None, min_total_weight=1, batch_size=10_000, *, client=None):
    """
    Fetch synaptic-level connections between source and target neurons.

    Note:
        Use this function if you need information about individual synapse connections,
        such as their exact positions or confidence scores.
        If you're just interested in aggregate neuron-to-neuron connection info
        (including connection strengths and ROI intersections), see
        :py:func:`fetch_simple_connections()` and :py:func:`fetch_adjacencies()`,
        which are faster and have more condensed outputs than this function.

    Note:
        If you experience timeouts while running this function,
        try reducing the ``batch_size`` setting.

    Args:

        source_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`):
            Criteria to by which to filter source (pre-synaptic) neurons.
            If omitted, all Neurons will be considered as possible sources.

            Note:
                Any ROI criteria specified in this argument does not affect
                which synapses are returned, only which bodies are inspected.

        target_criteria (bodyId(s), type/instance, or :py:class:`.NeuronCriteria`):
            Criteria to by which to filter target (post-synaptic) neurons.
            If omitted, all Neurons will be considered as possible targets.

            Note:
                Any ROI criteria specified in this argument does not affect
                which synapses are returned, only which bodies are inspected.

        synapse_criteria (SynapseCriteria):
            Optional. Allows you to filter synapses by roi, type, confidence.
            The same criteria is used to filter both ``pre`` and ``post`` sides
            of the connection.
            By default, ``SynapseCriteria(primary_only=True)`` is used.

            If ``primary_only`` is specified in the criteria, then the resulting
            ``roi_pre`` and ``roi_post`` columns will contain a single
            string (or ``None``) in every row.

            Otherwise, the roi columns will contain a list of ROIs for every row.
            (Primary ROIs do not overlap, so every synapse resides in only one
            (or zero) primary ROI.)
            See :py:class:`.SynapseCriteria` for details.

        min_total_weight:
            If the total weight of the connection between two bodies is not at least
            this strong, don't include the synapses for that connection in the results.

            Note:
                This filters for total connection weight, regardless of the weight
                within any particular ROI.  So, if your ``SynapseCriteria`` limits
                results to a particular ROI, but two bodies connect in multiple ROIs,
                then the number of synapses returned for the two bodies may appear to
                be less than ``min_total_weight``. That's because you filtered out
                the synapses in other ROIs.

        batch_size:
            To avoid timeouts and improve performance, the synapse connections
            will be fetched in batches.  The batching strategy is to process
            each body one at a time, and if it has lots of connection partners,
            split the request across several batches to avoid timeouts that
            could arise from a large request.
            This argument specifies the maximum size of each batch in the inner loop.
            Larger batches are more efficient to fetch, but increase the likelihood
            of timeouts.

        client:
            If not provided, the global default :py:class:`.Client` will be used.

    Returns:

        DataFrame in which each row represents a single synaptic connection
        between an upstream (pre-synaptic) body and downstream (post-synaptic) body.

        Synapse locations are listed in columns ``[x_pre, y_pre, z_pre]`` and
        ``[x_post, y_post, z_post]`` for the upstream and downstream synapses,
        respectively.

        The ``roi_pre`` and ``roi_post`` columns will contain either strings
        or lists-of-strings, depending on the ``primary_only`` synapse criteria as
        described above.

    Example:

        .. code-block:: ipython

            In [1]: from neuprint import fetch_synapse_connections, SynapseCriteria as SC
               ...: fetch_synapse_connections(792368888, None, SC(rois=['PED(R)', 'SMP(R)'], primary_only=True))
            Out[1]:
                bodyId_pre  bodyId_post roi_pre roi_post  x_pre  y_pre  z_pre  x_post  y_post  z_post  confidence_pre  confidence_post
            0    792368888    754547386  PED(R)   PED(R)  14013  27747  19307   13992   27720   19313           0.996         0.401035
            1    792368888    612742248  PED(R)   PED(R)  14049  27681  19417   14044   27662   19408           0.921         0.881487
            2    792368888   5901225361  PED(R)   PED(R)  14049  27681  19417   14055   27653   19420           0.921         0.436177
            3    792368888   5813117385  SMP(R)   SMP(R)  23630  29443  16297   23634   29437   16279           0.984         0.970746
            4    792368888   5813083733  SMP(R)   SMP(R)  23630  29443  16297   23634   29419   16288           0.984         0.933871
            5    792368888   5813058320  SMP(R)   SMP(R)  18662  34144  12692   18655   34155   12697           0.853         0.995000
            6    792368888   5812981989  PED(R)   PED(R)  14331  27921  20099   14351   27928   20085           0.904         0.877373
            7    792368888   5812981381  PED(R)   PED(R)  14331  27921  20099   14301   27919   20109           0.904         0.567321
            8    792368888   5812981381  PED(R)   PED(R)  14013  27747  19307   14020   27747   19285           0.996         0.697836
            9    792368888   5812979314  PED(R)   PED(R)  14331  27921  20099   14329   27942   20109           0.904         0.638362
            10   792368888    424767514  PED(R)   PED(R)  14331  27921  20099   14324   27934   20085           0.904         0.985734
            11   792368888    424767514  PED(R)   PED(R)  14013  27747  19307   14020   27760   19294           0.996         0.942831
            12   792368888    424767514  PED(R)   PED(R)  14049  27681  19417   14040   27663   19420           0.921         0.993586
            13   792368888    331662710  SMP(R)   SMP(R)  23630  29443  16297   23644   29429   16302           0.984         0.996389
            14   792368888   1196854070  PED(R)   PED(R)  14331  27921  20099   14317   27935   20101           0.904         0.968408
            15   792368888   1131831702  SMP(R)   SMP(R)  23630  29443  16297   23651   29434   16316           0.984         0.362952
    """
    assert source_criteria is not None or target_criteria is not None, \
        "Please specify either source or target search criteria (or both)."

    if synapse_criteria is None:
        synapse_criteria = SynapseCriteria(primary_only=True)

    def prepare_nc(nc, matchvar):
        nc.matchvar = matchvar

        # If the user specified rois to filter synapses by, but hasn't specified rois
        # in the NeuronCriteria, add them to the NeuronCriteria to speed up the query.
        if synapse_criteria.rois and not nc.rois:
            nc.rois = {*synapse_criteria.rois}
            nc.roi_req = 'any'

        return nc

    source_criteria = prepare_nc(source_criteria, 'n')
    target_criteria = prepare_nc(target_criteria, 'm')

    # Fetch the list of neuron-neuron pairs in advance so we can break into batches.
    _neuron_df, roi_conn_df = fetch_adjacencies( source_criteria,
                                                 target_criteria,
                                                 synapse_criteria.rois,
                                                 1,
                                                 min_total_weight,
                                                 properties=[] )

    if len(roi_conn_df) == 0:
        # Return empty dataframe, but with the correct dtypes
        dtypes = {
            'bodyId_pre': np.dtype('int64'),
            'bodyId_post': np.dtype('int64'),
            'roi_pre': np.dtype('O'),
            'roi_post': np.dtype('O'),
            'x_pre': np.dtype('int32'),
            'y_pre': np.dtype('int32'),
            'z_pre': np.dtype('int32'),
            'x_post': np.dtype('int32'),
            'y_post': np.dtype('int32'),
            'z_post': np.dtype('int32'),
            'confidence_pre': np.dtype('float32'),
            'confidence_post': np.dtype('float32')
        }
        return pd.DataFrame([], columns=dtypes.keys()).astype(dtypes)

    conn_df = (roi_conn_df.drop_duplicates(['bodyId_pre', 'bodyId_post'])
                          .sort_values(['bodyId_pre', 'bodyId_post']))

    # Pick either 'pre' or 'post' column to process bodies one at a time.
    # Then batch across the connections in an inner loop.
    num_pre = conn_df['bodyId_pre'].nunique()
    num_post = conn_df['bodyId_post'].nunique()
    if num_pre < num_post:
        grouping_col = 'bodyId_pre'
    else:
        grouping_col = 'bodyId_post'

    syn_dfs = []
    with tqdm(total=roi_conn_df['weight'].sum()) as progress:
        for _, group_df in conn_df.groupby(grouping_col):
            batches = iter_batches(group_df, batch_size)
            for batch_df in tqdm(batches, leave=False):
                src_crit = copy.copy(source_criteria)
                tgt_crit = copy.copy(target_criteria)

                if grouping_col == 'bodyId_pre':
                    assert batch_df['bodyId_pre'].nunique() == 1
                    src_crit.bodyId = batch_df['bodyId_pre'].unique()
                    # Filter target criteria further only if connections
                    # are being fetched in multiple batches.
                    # Otherwise, the explicit body list is unnecessary and slows down the query.
                    if len(batches) > 1:
                        tgt_crit.bodyId = batch_df['bodyId_post'].unique()
                else:
                    assert batch_df['bodyId_post'].nunique() == 1
                    tgt_crit.bodyId = batch_df['bodyId_post'].unique()
                    # Filter source criteria further only if connections
                    # are being fetched in multiple batches.
                    # Otherwise, the explicit body list is unnecessary and slows down the query.
                    if len(batches) > 1:
                        src_crit.bodyId = batch_df['bodyId_pre'].unique()

                batch_syn_df = _fetch_synapse_connections( src_crit,
                                                        tgt_crit,
                                                        synapse_criteria,
                                                        min_total_weight,
                                                        client )
                syn_dfs.append(batch_syn_df)
                progress.update(len(batch_syn_df))

    syn_df = pd.concat(syn_dfs, ignore_index=True)
    hashable_cols = [col for col, dtype in syn_df.dtypes.items() if dtype != object]
    assert syn_df.duplicated(hashable_cols).sum() == 0, \
        "Somehow obtained duplicate synapse-synapse connections!"
    return syn_df

def _fetch_synapse_connections(source_criteria, target_criteria, synapse_criteria, min_total_weight, client):
    if synapse_criteria.primary_only:
        return_rois = {*client.primary_rois}
    else:
        return_rois = {*client.all_rois}

    source_syn_crit = copy.copy(synapse_criteria)
    target_syn_crit = copy.copy(synapse_criteria)

    source_syn_crit.matchvar = 'ns'
    target_syn_crit.matchvar = 'ms'

    source_syn_crit.type = 'pre'
    target_syn_crit.type = 'post'

    criteria_globals = [*source_criteria.global_vars().keys(), *target_criteria.global_vars().keys()]
    combined_conditions = NeuronCriteria.combined_conditions(
        [source_criteria, target_criteria],
        ['n', 'e', 'm', 'ns', 'ms', *criteria_globals],
        prefix=8)

    # Fetch results
    cypher = dedent(f"""\
        {NeuronCriteria.combined_global_with((source_criteria, target_criteria), prefix=8)}
        MATCH (n:{source_criteria.label})-[e:ConnectsTo]->(m:{target_criteria.label}),
              (n)-[:Contains]->(nss:SynapseSet)-[:ConnectsTo]->(mss:SynapseSet)<-[:Contains]-(m),
              (nss)-[:Contains]->(ns:Synapse)-[:SynapsesTo]->(ms:Synapse)<-[:Contains]-(mss)

        {combined_conditions}

        // Note: Semantically, the word 'DISTINCT' is unnecessary here,
        // but its presence makes this query run faster.
        WITH DISTINCT n, m, ns, ms, e
        WHERE e.weight >= {min_total_weight}

        {source_syn_crit.condition('n', 'm', 'ns', 'ms', prefix=8)}
        {target_syn_crit.condition('n', 'm', 'ns', 'ms', prefix=8)}

        RETURN n.bodyId as bodyId_pre,
               m.bodyId as bodyId_post,
               ns.location.x as ux,
               ns.location.y as uy,
               ns.location.z as uz,
               ms.location.x as dx,
               ms.location.y as dy,
               ms.location.z as dz,
               ns.confidence as confidence_pre,
               ms.confidence as confidence_post,
               apoc.map.removeKeys(ns, ['location', 'confidence', 'type']) as info_pre,
               apoc.map.removeKeys(ms, ['location', 'confidence', 'type']) as info_post
    """)
    data = client.fetch_custom(cypher, format='json')['data']

    # Assemble DataFrame
    syn_table = []
    for bodyId_pre, bodyId_post, ux, uy, uz, dx, dy, dz, up_conf, dn_conf, info_pre, info_post in data:
        # Exclude non-primary ROIs if necessary
        pre_rois = return_rois & {*info_pre.keys()}
        post_rois = return_rois & {*info_post.keys()}

        # Intern the ROIs to save RAM
        pre_rois = sorted(map(sys.intern, pre_rois))
        post_rois = sorted(map(sys.intern, post_rois))

        pre_rois = pre_rois or [None]
        post_rois = post_rois or [None]

        # Should be (at most) one ROI when primary_only=True,
        # so only show that one (not a list)
        if synapse_criteria.primary_only:
            pre_rois = pre_rois[0]
            post_rois = post_rois[0]

        syn_table.append((bodyId_pre, bodyId_post, pre_rois, post_rois, ux, uy, uz, dx, dy, dz, up_conf, dn_conf))

    syn_df = pd.DataFrame(syn_table, columns=['bodyId_pre', 'bodyId_post',
                                              'roi_pre', 'roi_post',
                                              'x_pre', 'y_pre', 'z_pre', 'x_post', 'y_post', 'z_post',
                                              'confidence_pre', 'confidence_post'])

    # Save RAM with smaller dtypes
    syn_df['x_pre'] = syn_df['x_pre'].astype(np.int32)
    syn_df['y_pre'] = syn_df['y_pre'].astype(np.int32)
    syn_df['z_pre'] = syn_df['z_pre'].astype(np.int32)
    syn_df['x_post'] = syn_df['x_post'].astype(np.int32)
    syn_df['y_post'] = syn_df['y_post'].astype(np.int32)
    syn_df['z_post'] = syn_df['z_post'].astype(np.int32)
    syn_df['confidence_pre'] = syn_df['confidence_pre'].astype(np.float32)
    syn_df['confidence_post'] = syn_df['confidence_post'].astype(np.float32)

    return syn_df

max_stars_repo_name: umayaml/neuprint-python

max_issues_repo_name: umayaml/neuprint-python

max_forks_repo_name: umayaml/neuprint-python


text:
from os import path
from typing import List

import matplotlib.pyplot as plt
import numpy as np
from torch.utils.data import DataLoader, Dataset, Subset
from torchvision import transforms
from torchvision.datasets import MNIST, CIFAR10, ImageFolder, CIFAR100, FashionMNIST

from data import DataManager

from configuration import config
from src.phenotype.augmentations.augmentation_scheme import AugmentationScheme

def load_data(composed_transforms: transforms.Compose, split: str) -> DataLoader:
    """
    Loads the data given the config.dataset

    :param composed_transforms: the augmentations to apply to the dataset
    :param split: either train or test, dataset returned also depends on config.fully_train -
    'train' will return 42500 images for evolution, but 50000 for fully training.
    'test' will return 7500 images for evolution, but 10000 for fully training.

    Note: the validation/train split does not try balance the classes of data, it just takes the first n for the train
    set and the remaining data goes to the validation set
    """
    if split.lower() not in ['train', 'test']:
        raise ValueError('Parameter split can be one of train, test or validation, but received: ' + str(split))

    # when to load train set
    train: bool = True if split == 'train' or (split == 'test' and not config.fully_train) else False

    dataset_args = {
        'root': DataManager.get_datasets_folder(),
        'train': train,
        'download': config.download_dataset,
        'transform': composed_transforms
    }

    if config.dataset == 'mnist':
        dataset = MNIST(**dataset_args)
    elif config.dataset == 'cifar10':
        dataset = CIFAR10(**dataset_args)
    elif config.dataset == 'cifar100':
        dataset = CIFAR100(**dataset_args)
    elif config.dataset == 'fashionMnist':
        dataset = FashionMNIST(**dataset_args)
    elif config.dataset == 'custom':
        dataset = get_generic_dataset(composed_transforms, train)
    else:
        raise ValueError('config.dataset can be one of mnist, cifar10 or custom, but received: ' + str(config.dataset))

    if train and not config.fully_train:
        # Splitting the train set into a train and valid set
        train_size = int(len(dataset) * (1 - config.validation_split))
        if split == 'train':
            dataset = Subset(dataset, range(train_size))
        elif split == 'test':
            dataset = Subset(dataset, range(train_size, len(dataset)))
        else:
            raise Exception("")

    # print(split, 'set size in', 'FT' if config.fully_train else 'evo', len(dataset))

    # TODO: test num workers and pin memory
    return DataLoader(dataset, batch_size=config.batch_size, shuffle=True, num_workers=0, pin_memory=False)

def get_data_shape() -> List[int]:
    return list(next(iter(load_data(load_transform(), 'test')))[0].size())

def get_generic_dataset(composed_transforms: transforms.Compose, train: bool) -> Dataset:
    """
    Loads data from custom_dataset_root, given that the data is structured as follows:
    root/train/dog/xxx.png
    root/test/dog/xxy.png

    root/train/cat/123.png
    root/test/cat/nsdf3.png

    Note: the train set should contain enough data to be split into a train and validation set
    :return: a train and test dataloader
    """
    if train:
        data = ImageFolder(root=path.join(config.custom_dataset_root, 'train'), transform=composed_transforms)
    else:
        data = ImageFolder(root=path.join(config.custom_dataset_root, 'test'), transform=composed_transforms)

    return data

def load_transform(aug: AugmentationScheme = None) -> transforms.Compose:
    norm_tuple = (0.5, 0.5, 0.5) if config.use_colour_augmentations else (0.5,)

    if config.evolve_da and not config.batch_augmentation and aug is not None:
        return transforms.Compose([
            aug,
            transforms.ToTensor(),
            transforms.Normalize(norm_tuple, norm_tuple)
        ])
    else:
        return transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(norm_tuple, norm_tuple)
        ])

def imshow(img):
    img = img / 2 + 0.5  # unnormalize
    npimg = img.cpu().numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

max_stars_repo_name: sash-a/CoDeepNEAT

max_issues_repo_name: sash-a/CoDeepNEAT

max_forks_repo_name: sash-a/CoDeepNEAT