text:
from utils import art_mean_workflow

def get_b0(bvals):
    import numpy as np
    vals = np.genfromtxt(bvals)
    b0 = np.asarray(range(0,vals.shape[0]))[vals<1]
    b0=b0.tolist()
    print b0
    return b0

def get_params(A):
    """This is a copy of spm's spm_imatrix where

    we already know the rotations and translations matrix,
    shears and zooms (as outputs from fsl FLIRT/avscale)

    Let A = the 4x4 rotation and translation matrix

    R = [          c5*c6,           c5*s6, s5]
        [-s4*s5*c6-c4*s6, -s4*s5*s6+c4*c6, s4*c5]
        [-c4*s5*c6+s4*s6, -c4*s5*s6-s4*c6, c4*c5]

    """
    import numpy as np

    def rang(b):
        a = min(max(b, -1), 1)
        return a
    Ry = np.arcsin(A[0,2])
    #Rx = np.arcsin(A[1,2]/np.cos(Ry))
    #Rz = np.arccos(A[0,1]/np.sin(Ry))

    if (abs(Ry)-np.pi/2)**2 < 1e-9:
        Rx = 0
        Rz = np.arctan2(-rang(A[1,0]), rang(-A[2,0]/A[0,2]))
    else:
        c  = np.cos(Ry)
        Rx = np.arctan2(rang(A[1,2]/c), rang(A[2,2]/c))
        Rz = np.arctan2(rang(A[0,1]/c), rang(A[0,0]/c))

    rotations = [Rx, Ry, Rz]
    translations = [A[0,3], A[1,3], A[2,3]]

    return rotations, translations

def get_flirt_motion_parameters(flirt_out_mats):
    import nipype.interfaces.fsl.utils as fsl
    import os
    import numpy as np
    from bips.workflows.scripts.u0a14c5b5899911e1bca80023dfa375f2.modified_nipype_workflows import get_params
    motion_params = open(os.path.abspath('motion_parameters.par'),'w')
    for mat in flirt_out_mats:
        res = fsl.AvScale(mat_file = mat).run()
        A = np.asarray(res.outputs.rotation_translation_matrix)
        rotations, translations = get_params(A)
        for i in rotations+translations:
            motion_params.write('%f '%i)
        motion_params.write('\n')
    motion_params.close()
    motion_params = os.path.abspath('motion_parameters.par')
    return motion_params

def mapnode_coregistration(in_file, reference):
    import nipype.interfaces.fsl as fsl
    out_file = []
    out_matrix_file = []
    for f in in_file:
        coreg = fsl.FLIRT(no_search=True,
                          padding_size=1,
                          dof=9,
                          in_file = f,
                          reference= reference)
        res = coreg.run()
        out_file.append(res.outputs.out_file)
        out_matrix_file.append(res.outputs.out_matrix_file)
    return out_file, out_matrix_file

def create_eddy_correct_pipeline(name="eddy_correct"):
    """Creates a pipeline that replaces eddy_correct script in FSL. It takes a
    series of diffusion weighted images and linearly corregisters them to one
    reference image.

    Example
    -------

    >>> nipype_eddycorrect = create_eddy_correct_pipeline("nipype_eddycorrect")
    >>> nipype_eddycorrect.inputs.inputnode.in_file = 'diffusion.nii'
    >>> nipype_eddycorrect.inputs.inputnode.ref_num = 0
    >>> nipype_eddycorrect.run() # doctest: +SKIP

    Inputs::

    inputnode.in_file
    inputnode.ref_num

    Outputs::

    outputnode.eddy_corrected
    """
    import nipype.pipeline.engine as pe
    import nipype.interfaces.utility as util
    import nipype.interfaces.fsl as fsl
    inputnode = pe.Node(interface = util.IdentityInterface(fields=["in_file", "bvals"]),
        name="inputnode")

    pipeline = pe.Workflow(name=name)

    split = pe.MapNode(fsl.Split(dimension='t'), name="split", iterfield=['in_file'])
    pipeline.connect([(inputnode, split, [("in_file", "in_file")])])

    getb0 = pe.Node(util.Function(input_names=['bvals'],
                                    output_names=['b0'],
                                    function=get_b0),
        name='get_b0_indices')

    pick_ref = pe.MapNode(util.Select(), name="pick_b0_scans", iterfield=["inlist"])
    pick0 = pe.MapNode(util.Select(index=0), name="pick_first_scan", iterfield=["inlist"])

    getmotion = pe.MapNode(util.Function(input_names=['flirt_out_mats'],
                                         output_names=['motion_params'],
                                         function=get_flirt_motion_parameters),
        name='getmotion', iterfield=['flirt_out_mats'])

    pipeline.connect(split, 'out_files', pick_ref, "inlist")
    pipeline.connect(split, 'out_files', pick0, "inlist")
    pipeline.connect(inputnode,'bvals', getb0, 'bvals')
    pipeline.connect(getb0, 'b0', pick_ref, "index")
    #pipeline.connect([(split, pick_ref, [("out_files", "inlist")]),
    #    (inputnode, pick_ref, [("ref_num", "index")])])

    coregistration = pe.MapNode(util.Function(input_names=['in_file', 'reference'],
        output_names=['out_file', 'out_matrix_file'],
        function = mapnode_coregistration),
        name = "coregister_b0",
        iterfield=["in_file","reference"])
    pipeline.connect([(pick_ref, coregistration, [("out", "in_file")]),
        (pick0, coregistration, [("out", "reference")])])

    art_mean = art_mean_workflow()
    art_mean.inputs.inputspec.parameter_source='FSL'

    pipeline.connect(coregistration,'out_matrix_file', getmotion, 'flirt_out_mats')
    pipeline.connect(getmotion, 'motion_params', art_mean, 'inputspec.realignment_parameters')

    merge = pe.MapNode(fsl.Merge(dimension="t"), name="merge", iterfield=["in_files"])
    pipeline.connect([(coregistration, merge, [("out_file", "in_files")])
    ])

    pipeline.connect(merge,'merged_file',art_mean,'inputspec.realigned_files')

    outputnode = pe.Node(interface = util.IdentityInterface(fields=["eddy_corrected","mean_image","coreg_mat_files"]),
        name="outputnode")

    pipeline.connect(art_mean, 'outputspec.mean_image', outputnode, "mean_image")

    coreg_all = coregistration.clone("coregister_all")
    pipeline.connect(split, 'out_files', coreg_all, 'in_file')
    pipeline.connect(art_mean,'outputspec.mean_image',coreg_all,"reference")

    merge_all = merge.clone("merge_all")
    pipeline.connect([(coreg_all, merge_all, [("out_file", "in_files")])
    ])
    pipeline.connect(coreg_all, 'out_matrix_file', outputnode, "coreg_mat_files")
    pipeline.connect(merge_all,'merged_file',outputnode,"eddy_corrected")

    return pipeline

"""
Connect the nodes
"""
def get_aparc_aseg(files):
    for name in files:
        if 'aparc+aseg' in name:
            return name
    raise ValueError('aparc+aseg.mgz not found')

def create_getmask_flow(name='getmask', dilate_mask=True):
    """Registers a source file to freesurfer space and create a brain mask in
source space

Requires fsl tools for initializing registration

Parameters
----------

name : string
name of workflow
dilate_mask : boolean
indicates whether to dilate mask or not

Example
-------

>>> getmask = create_getmask_flow()
>>> getmask.inputs.inputspec.source_file = 'mean.nii'
>>> getmask.inputs.inputspec.subject_id = 's1'
>>> getmask.inputs.inputspec.subjects_dir = '.'
>>> getmask.inputs.inputspec.contrast_type = 't2'

Inputs::

inputspec.source_file : reference image for mask generation
inputspec.subject_id : freesurfer subject id
inputspec.subjects_dir : freesurfer subjects directory
inputspec.contrast_type : MR contrast of reference image

Outputs::

outputspec.mask_file : binary mask file in reference image space
outputspec.reg_file : registration file that maps reference image to
freesurfer space
outputspec.reg_cost : cost of registration (useful for detecting misalignment)
"""
    import nipype.pipeline.engine as pe
    import nipype.interfaces.utility as niu
    import nipype.interfaces.freesurfer as fs
    import nipype.interfaces.io as nio
    """
Initialize the workflow
"""

    getmask = pe.Workflow(name=name)

    """
Define the inputs to the workflow.
"""

    inputnode = pe.Node(niu.IdentityInterface(fields=['source_file',
                                                      'subject_id',
                                                      'subjects_dir',
                                                      'contrast_type']),
        name='inputspec')

    """
Define all the nodes of the workflow:

fssource: used to retrieve aseg.mgz
threshold : binarize aseg
register : coregister source file to freesurfer space
voltransform: convert binarized aseg to source file space

"""

    fssource = pe.Node(nio.FreeSurferSource(),
        name = 'fssource')
    threshold = pe.Node(fs.Binarize(min=0.5, out_type='nii'),
        name='threshold')
    register = pe.MapNode(fs.BBRegister(init='fsl'),
        iterfield=['source_file'],
        name='register')
    voltransform = pe.MapNode(fs.ApplyVolTransform(inverse=True),
        iterfield=['source_file', 'reg_file'],
        name='transform')

    """
Connect the nodes
"""

    getmask.connect([
        (inputnode, fssource, [('subject_id','subject_id'),
            ('subjects_dir','subjects_dir')]),
        (inputnode, register, [('source_file', 'source_file'),
            ('subject_id', 'subject_id'),
            ('subjects_dir', 'subjects_dir'),
            ('contrast_type', 'contrast_type')]),
        (inputnode, voltransform, [('subjects_dir', 'subjects_dir'),
            ('source_file', 'source_file')]),
        (fssource, threshold, [(('aparc_aseg', get_aparc_aseg), 'in_file')]),
        (register, voltransform, [('out_reg_file','reg_file')]),
        (threshold, voltransform, [('binary_file','target_file')])
    ])

    """
Add remaining nodes and connections

dilate : dilate the transformed file in source space
threshold2 : binarize transformed file
"""

    threshold2 = pe.MapNode(fs.Binarize(min=0.5, out_type='nii'),
        iterfield=['in_file'],
        name='threshold2')
    if dilate_mask:
        threshold2.inputs.dilate = 1
    getmask.connect([
        (voltransform, threshold2, [('transformed_file', 'in_file')])
    ])

    """
Setup an outputnode that defines relevant inputs of the workflow.
"""

    outputnode = pe.Node(niu.IdentityInterface(fields=["mask_file",
                                                       "reg_file",
                                                       "reg_cost"
    ]),
        name="outputspec")
    getmask.connect([
        (register, outputnode, [("out_reg_file", "reg_file")]),
        (register, outputnode, [("min_cost_file", "reg_cost")]),
        (threshold2, outputnode, [("binary_file", "mask_file")]),
    ])
    return getmask

def create_get_stats_flow(name='getstats', withreg=False):
    """Retrieves stats from labels

    Parameters
    ----------

    name : string
        name of workflow
    withreg : boolean
        indicates whether to register source to label

    Example
    -------

    Inputs::

           inputspec.source_file : reference image for mask generation
           inputspec.label_file : label file from which to get ROIs

           (optionally with registration)
           inputspec.reg_file : bbreg file (assumes reg from source to label
           inputspec.inverse : boolean whether to invert the registration
           inputspec.subjects_dir : freesurfer subjects directory

    Outputs::

           outputspec.stats_file : stats file
    """
    import nipype.pipeline.engine as pe
    import nipype.interfaces.freesurfer as fs
    import nipype.interfaces.utility as niu

    """
    Initialize the workflow
    """

    getstats = pe.Workflow(name=name)

    """
    Define the inputs to the workflow.
    """

    if withreg:
        inputnode = pe.Node(niu.IdentityInterface(fields=['source_file',
                                                          'label_file',
                                                          'reg_file',
                                                          'subjects_dir','inverse']),
            name='inputspec')
    else:
        inputnode = pe.Node(niu.IdentityInterface(fields=['source_file',
                                                          'label_file']),
            name='inputspec')

    statnode = pe.MapNode(fs.SegStats(),
        iterfield=['segmentation_file','in_file'],
        name='segstats')

    """
    Convert between source and label spaces if registration info is provided

    """
    if withreg:
        voltransform = pe.MapNode(fs.ApplyVolTransform(inverse=True, interp='nearest'),
            iterfield=['source_file', 'reg_file'],
            name='transform')
        getstats.connect(inputnode, 'reg_file', voltransform, 'reg_file')
        getstats.connect(inputnode, 'source_file', voltransform, 'source_file')
        getstats.connect(inputnode, 'label_file', voltransform, 'target_file')
        getstats.connect(inputnode, 'subjects_dir', voltransform, 'subjects_dir')

        def switch_labels(inverse, transform_output, source_file, label_file):
            if inverse:
                return transform_output, source_file
            else:
                return label_file, transform_output

        chooser = pe.MapNode(niu.Function(input_names = ['inverse',
                                                         'transform_output',
                                                         'source_file',
                                                         'label_file'],
            output_names = ['label_file',
                            'source_file'],
            function=switch_labels),
            iterfield=['transform_output','source_file'],
            name='chooser')
        getstats.connect(inputnode,'source_file', chooser, 'source_file')
        getstats.connect(inputnode,'label_file', chooser, 'label_file')
        getstats.connect(inputnode,'inverse', chooser, 'inverse')
        getstats.connect(voltransform, 'transformed_file', chooser, 'transform_output')
        getstats.connect(chooser, 'label_file', statnode, 'segmentation_file')
        getstats.connect(chooser, 'source_file', statnode, 'in_file')
    else:
        getstats.connect(inputnode, 'label_file', statnode, 'segmentation_file')
        getstats.connect(inputnode, 'source_file', statnode, 'in_file')

    """
    Setup an outputnode that defines relevant inputs of the workflow.
    """

    outputnode = pe.Node(niu.IdentityInterface(fields=["stats_file"
    ]),
        name="outputspec")
    getstats.connect([
        (statnode, outputnode, [("summary_file", "stats_file")]),
    ])
    return getstats

max_stars_repo_name: INCF/BrainImagingPipelines

max_issues_repo_name: akeshavan/BrainImagingPipelines

max_forks_repo_name: akeshavan/BrainImagingPipelines


text:
import sys
import cuzcatlan as cusca
import numpy as np
from statistics import mode
from scipy.cluster.hierarchy import dendrogram
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import itertools

SIGNIFICANT_DIGITS = 7

# This dictionary is used for parsing inputs
input_col_distance_dict = {
    # These are the values I expect
    "No column clustering": "No_column_clustering",
    "Uncentered correlation": "uncentered_pearson",
    "Pearson correlation": "pearson",
    "Uncentered correlation, absolute value": "absolute_uncentered_pearson",
    "Pearson correlation, absolute value": "absolute_pearson",
    "Spearman's rank correlation": "spearman",
    "Kendall's tau": "kendall",
    "Euclidean distance": "euclidean",
    "City-block distance": "manhattan",
    "No_column_clustering": "No_column_clustering",
    # These are the values the GpUnit tests give
    "0": "No_column_clustering",
    "1": "uncentered_pearson",
    "2": "pearson",
    "3": "absolute_uncentered_pearson",
    "4": "absolute_pearson",
    "5": "spearman",
    "6": "kendall",
    "7": "euclidean",
    "8": "manhattan",
    "9": "information_coefficient",
    # These are the values I expect from the comand line
    "no_col": "No_column_clustering",
    "uncentered_pearson": "uncentered_pearson",
    "pearson": "pearson",
    "absolute_uncentered_pearson": "absolute_uncentered_pearson",
    "absolute_pearson": "absolute_pearson",
    "spearman": "spearman",
    "kendall": "kendall",
    "euclidean": "euclidean",
    "manhattan": "manhattan",
    "Cosine": "cosine",
    "cosine": "cosine",
    "ic": "information_coefficient",
    "information_coefficient": "information_coefficient",
    "Information Coefficient": "information_coefficient",
}

# This dictionary is used for parsing inputs
input_row_distance_dict = {
    # These are the values I expect
    "No row clustering": "No_row_clustering",
    "Uncentered correlation": "uncentered_pearson",
    "Pearson correlation": "pearson",
    "Uncentered correlation, absolute value": "absolute_uncentered_pearson",
    "Pearson correlation, absolute value": "absolute_pearson",
    "Spearman's rank correlation": "spearman",
    "Kendall's tau": "kendall",
    "Euclidean distance": "euclidean",
    "City-block distance": "manhattan",
    "No_row_clustering": "No_row_clustering",
    # These are the values the GpUnit tests give
    "0": "No_row_clustering",
    "1": "uncentered_pearson",
    "2": "pearson",
    "3": "absolute_uncentered_pearson",
    "4": "absolute_pearson",
    "5": "spearman",
    "6": "kendall",
    "7": "euclidean",
    "8": "manhattan",
    "9": "information_coefficient",
    # These are the values I expect from the comand line
    "no_row": "No_row_clustering",
    "uncentered_pearson": "uncentered_pearson",
    "pearson": "pearson",
    "absolute_uncentered_pearson": "absolute_uncentered_pearson",
    "absolute_pearson": "absolute_pearson",
    "spearman": "spearman",
    "kendall": "kendall",
    "euclidean": "euclidean",
    "manhattan": "manhattan",
    "Cosine": "cosine",
    "cosine": "cosine",
    "ic": "information_coefficient",
    "information_coefficient": "information_coefficient",
    "Information Coefficient": "information_coefficient",
}

# This dictionary is used for parsing inputs
input_clustering_method = {
    # These are the values I expect
    'Pairwise complete-linkage': 'complete',
    'Pairwise average-linkage': 'average',
    'Pairwise ward-linkage': 'ward',
    # These are the values the GpUnit test give
    'm': 'complete',
    'a': 'average',  # I think this is the default
}

# This dictionary is used for parsing inputs
input_row_centering = {
    # These are the values I expect
    'No': None,
    'Subtract the mean from each row': None,
    'Subtract the median from each row': None,
    # These are the values the GpUnit test give
    'None': None,
    'Median': 'Median',
    'Mean': 'Mean',
}

# This dictionary is used for parsing inputs
input_row_normalize = {
    # These are the values I expect
    'No': False,
    'Yes': True,
    # These are the values the GpUnit test give
    'False': False,
    'True': True,
}

# This dictionary is used for parsing inputs
input_col_centering = {
    # These are the values I expect
    'No': None,
    'Subtract the mean from each column': None,
    'Subtract the median from each column': None,
    # These are the values the GpUnit test give
    'None': None,
    'Median': 'Median',
    'Mean': 'Mean',
}

# This dictionary is used for parsing inputs
input_col_normalize = {
    # These are the values I expect
    'No': False,
    'Yes': True,
    # These are the values the GpUnit test give
    'False': False,
    'True': True,
}

def parse_inputs(args=sys.argv):
    """
    This is the function which parses the inputs, it should be replaced with argparse
    :param args:
    :return:
    """
    # inp = []
    # inp = args
    # Error handling:
    arg_n = len(args)
    if arg_n == 1:
        sys.exit("Not enough parameters files were provided. This module needs a GCT file to work.")
    elif arg_n == 2:
        gct_name = args[1]
        col_distance_metric = 'euclidean'
        output_distances = False
        row_distance_metric = 'No_row_clustering'
        clustering_method = 'Pairwise average-linkage'
        output_base_name = 'HC_out'
        row_normalization = False
        col_normalization = False
        row_centering = None
        col_centering = None
        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric = euclidean (default value)")
        print("\toutput_distances =", output_distances, "(default: not computing it and creating a file)")
        print("\trow_distance_metric =", row_distance_metric, "(default: No row clustering)")
        print("\tclustering_method =", clustering_method, "(default: Pairwise average-linkage)")
        print("\toutput_base_name =", output_base_name, "(default: HC_out)")
        print("\trow_normalization =", row_normalization, "(default: False)")
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 3:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = False
        row_distance_metric = 'No_row_clustering'
        clustering_method = 'Pairwise average-linkage'
        output_base_name = 'HC_out'
        row_normalization = False
        col_normalization = False
        row_centering = None
        col_centering = None
        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", input_col_distance_dict[col_distance_metric])
        print("\toutput_distances =", output_distances, "(default: not computing it and creating a file)")
        print("\trow_distance_metric =", row_distance_metric, "(default: No row clustering)")
        print("\tclustering_method =", clustering_method, "(default: Pairwise average-linkage)")
        print("\toutput_base_name =", output_base_name, "(default: HC_out)")
        print("\trow_normalization =", row_normalization, "(default: False)")
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 4:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = 'No_row_clustering'
        clustering_method = 'Pairwise average-linkage'
        output_base_name = 'HC_out'
        row_normalization = False
        col_normalization = False
        row_centering = None
        col_centering = None

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        if (output_distances == 'False') or (output_distances == 'F')\
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True
        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric, "(default: No row clustering)")
        print("\tclustering_method =", clustering_method, "(default: Pairwise average-linkage)")
        print("\toutput_base_name =", output_base_name, "(default: HC_out)")
        print("\trow_normalization =", row_normalization, "(default: False)")
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 5:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = 'Pairwise average-linkage'
        # clustering_method = 'Pairwise complete-linkage'
        output_base_name = 'HC_out'
        row_normalization = False
        col_normalization = False
        row_centering = None
        col_centering = None

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method, "(default: Pairwise average-linkage)")
        print("\toutput_base_name =", output_base_name, "(default: HC_out)")
        print("\trow_normalization =", row_normalization, "(default: False)")
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 6:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = args[5]

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        clustering_method = input_clustering_method[clustering_method]
        if clustering_method not in linkage_dic.keys():
            exit("Clustering method chosen not supported. This should not have happened.")

        if (linkage_dic[clustering_method] == 'ward') and (col_distance_metric != 'average'):
            exit("When choosing 'Pairwise ward-linkage' the distance metric *must* be 'average' ")

        output_base_name = 'HC_out'
        row_normalization = False
        col_normalization = False
        row_centering = None
        col_centering = None
        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method)
        print("\toutput_base_name =", output_base_name, "(default: HC_out)")
        print("\trow_normalization =", row_normalization, "(default: False)")
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 7:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = args[5]
        output_base_name = args[6]
        row_normalization = False
        col_normalization = False
        row_centering = None
        col_centering = None

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        clustering_method = input_clustering_method[clustering_method]
        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method)
        print("\toutput_base_name =", output_base_name)
        print("\trow_normalization =", row_normalization, "(default: False)")
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 8:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = args[5]
        output_base_name = args[6]
        row_normalization = args[7]
        col_normalization = False
        row_centering = None
        col_centering = None

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        clustering_method = input_clustering_method[clustering_method]
        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        row_normalization = input_row_normalize[row_normalization]
        # if (row_normalization == 'False') or (row_normalization == 'F') \
        #         or (row_normalization == 'false') or (row_normalization == 'f'):
        #     row_normalization = False
        # else:
        #     row_normalization = True

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method)
        print("\toutput_base_name =", output_base_name)
        print("\trow_normalization =", row_normalization)
        print("\tcol_normalization =", col_normalization, "(default: False)")
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 9:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = args[5]
        output_base_name = args[6]
        row_normalization = args[7]
        col_normalization = args[8]
        row_centering = None
        col_centering = None

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        clustering_method = input_clustering_method[clustering_method]
        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        # Row normalization
        row_normalization = input_row_normalize[row_normalization]
        # if (row_normalization == 'False') or (row_normalization == 'F') \
        #         or (row_normalization == 'false') or (row_normalization == 'f'):
        #     row_normalization = False
        # else:
        #     row_normalization = True

        # Column normalization
        col_normalization = input_col_normalize[col_normalization]
        # if (col_normalization == 'False') or (col_normalization == 'F') \
        #         or (col_normalization == 'false') or (col_normalization == 'f'):
        #     col_normalization = False
        # else:
        #     col_normalization = True

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method)
        print("\toutput_base_name =", output_base_name)
        print("\trow_normalization =", row_normalization)
        print("\tcol_normalization =", col_normalization)
        print("\trow_centering =", row_centering, "(default: None)")
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 10:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = args[5]
        output_base_name = args[6]
        row_normalization = args[7]
        col_normalization = args[8]
        row_centering = args[9]
        col_centering = None

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        clustering_method = input_clustering_method[clustering_method]

        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        # Row normalization
        row_normalization = input_row_normalize[row_normalization]
        # if (row_normalization == 'False') or (row_normalization == 'F') \
        #         or (row_normalization == 'false') or (row_normalization == 'f'):
        #     row_normalization = False
        # else:
        #     row_normalization = True

        # Column normalization
        col_normalization = input_col_normalize[col_normalization]
        # if (col_normalization == 'False') or (col_normalization == 'F') \
        #         or (col_normalization == 'false') or (col_normalization == 'f'):
        #     col_normalization = False
        # else:
        #     col_normalization = True

        # row_centering
        row_centering = input_row_centering[row_centering]
        if (row_centering == 'None') or (col_normalization == 'N') \
                or (row_centering == 'none') or (col_normalization == 'n'):
            col_normalization = None

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method)
        print("\toutput_base_name =", output_base_name)
        print("\trow_normalization =", row_normalization)
        print("\tcol_normalization =", col_normalization)
        print("\trow_centering =", row_centering)
        print("\tcol_centering =", col_centering, "(default: None)")
    elif arg_n == 11:
        gct_name = args[1]
        col_distance_metric = args[2]
        output_distances = args[3]
        row_distance_metric = args[4]
        clustering_method = args[5]
        output_base_name = args[6]
        row_normalization = args[7]
        col_normalization = args[8]
        row_centering = args[9]
        col_centering = args[10]

        col_distance_metric = input_col_distance_dict[col_distance_metric]
        row_distance_metric = input_row_distance_dict[row_distance_metric]
        clustering_method = input_clustering_method[clustering_method]

        if (output_distances == 'False') or (output_distances == 'F') \
                or (output_distances == 'false') or (output_distances == 'f'):
            output_distances = False
        else:
            output_distances = True

        # Row normalization
        row_normalization = input_row_normalize[row_normalization]
        # if (row_normalization == 'False') or (row_normalization == 'F') \
        #         or (row_normalization == 'false') or (row_normalization == 'f'):
        #     row_normalization = False
        # else:
        #     row_normalization = True

        # Column normalization
        col_normalization = input_col_normalize[col_normalization]
        # if (col_normalization == 'False') or (col_normalization == 'F') \
        #         or (col_normalization == 'false') or (col_normalization == 'f'):
        #     col_normalization = False
        # else:
        #     col_normalization = True

        # row_centering
        row_centering = input_row_centering[row_centering]
        if (row_centering == 'None') or (col_normalization == 'N') \
                or (row_centering == 'none') or (col_normalization == 'n'):
            col_normalization = None

        # col_centering
        col_centering = input_col_centering[col_centering]
        if (col_centering == 'None') or (col_centering == 'N') \
                or (col_centering == 'none') or (col_centering == 'n'):
            col_centering = None

        print("Using:")
        print("\tgct_name =", gct_name)
        print("\tcol_distance_metric =", col_distance_metric)
        print("\toutput_distances =", output_distances)
        print("\trow_distance_metric =", row_distance_metric)
        print("\tclustering_method =", clustering_method)
        print("\toutput_base_name =", output_base_name)
        print("\trow_normalization =", row_normalization)
        print("\tcol_normalization =", col_normalization)
        print("\trow_centering =", row_centering)
        print("\tcol_centering =", col_centering)
    else:
        sys.exit("Too many inputs. This module needs only a GCT file to work, "
                 "plus an optional input choosing between Pearson Correlation or Information Coefficient.")

    print(args)
    return gct_name, col_distance_metric, output_distances, row_distance_metric, clustering_method, output_base_name, \
        row_normalization, col_normalization, row_centering, col_centering

def plot_dendrogram(model, data, tree, axis, dist=cusca.mydist, title='no_title.png', **kwargs):
    """
    #modified from https://github.com/scikit-learn/scikit-learn/pull/3464/files
    :param model:
    :param data:
    :param tree:
    :param axis:
    :param dist:
    :param title:
    :param kwargs:
    :return:
    """
    plt.clf()

    # Children of hierarchical clustering
    children = model.children_

    # Distances between each pair of children
    distance = better_dendodist(children, dist, tree, data, axis=axis)

    # norm_distances = []
    # for value in distance:
    #     norm_distances.append(1/value)
    # norm_distances = distance

    list_of_children = list(get_children(tree, leaves_are_self_children=False).values())
    no_of_observations = [len(i) for i in list_of_children if i]
    no_of_observations.append(len(no_of_observations)+1)

    if all(value == 0 for value in distance):
        # If all distances are zero, then use uniform distance
        distance = np.arange(len(distance))

    # The number of observations contained in each cluster level
    # no_of_observations = np.arange(2, children.shape[0]+2)
    # print(no_of_observations)

    #  # Create linkage matrix and then plot the dendrogram
    # linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
    # linkage_matrix = np.column_stack([children, np.cumsum(distance), no_of_observations]).astype(float)
    linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
    # linkage_matrix = np.column_stack([children, norm_distances, no_of_observations]).astype(float)
    # print(linkage_matrix)
    # Plot the corresponding dendrogram

    R = dendrogram(linkage_matrix, color_threshold=0, **kwargs)
    [label.set_rotation(90) for label in plt.gca().get_xticklabels()]
    order_of_columns = R['ivl']
    # # print(order_of_columns)
    # plt.gca().get_yaxis().set_visible(False)
    # plt.savefig(title, dpi=300)

    # n = len(linkage_matrix) + 1
    # cache = dict()
    # for k in range(len(linkage_matrix)):
    #     c1, c2 = int(linkage_matrix[k][0]), int(linkage_matrix[k][1])
    #     c1 = [c1] if c1 < n else cache.pop(c1)
    #     c2 = [c2] if c2 < n else cache.pop(c2)
    #     cache[n + k] = c1 + c2
    # order_of_columns = cache[2 * len(linkage_matrix)]

    return order_of_columns

def order_leaves(model, data, tree, labels, axis=0, dist=cusca.mydist, reverse=False):
    # Adapted from here: https://stackoverflow.com/questions/12572436/calculate-ordering-of-dendrogram-leaves

    children = model.children_
    # distance = better_dendodist(children, dist, tree, data, axis=axis)
    # if all(value == 0 for value in distance):
    #     distance = np.arange(len(distance))

    # list_of_children = list(get_children(tree, leaves_are_self_children=False).values())
    # no_of_observations = [len(i) for i in list_of_children if i]
    # no_of_observations.append(len(no_of_observations)+1)

    # Create linkage matrix and then plot the dendrogram
    # linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
    pseudo_linkage_matrix = np.column_stack([children]).astype(float)

    n = len(pseudo_linkage_matrix) + 1

    # This orders leaves by number of clusters
    cache = dict()
    for k in range(len(pseudo_linkage_matrix)):
        c1, c2 = int(pseudo_linkage_matrix[k][0]), int(pseudo_linkage_matrix[k][1])
        c1 = [c1] if c1 < n else cache.pop(c1)
        c2 = [c2] if c2 < n else cache.pop(c2)
        cache[n + k] = c1 + c2
    numeric_order_of_leaves = cache[2 * len(pseudo_linkage_matrix)]

    if reverse:
        numeric_order_of_leaves = list(reversed(numeric_order_of_leaves))

    return [labels[i] for i in numeric_order_of_leaves]

def two_plot_two_dendrogram(model, dist=cusca.mydist, **kwargs):
    """
    modified from https://github.com/scikit-learn/scikit-learn/pull/3464/files
    :param model:
    :param dist:
    :param kwargs:
    :return:
    """
    # Children of hierarchical clustering
    children = model.children_
    # Distances between each pair of children
    distance = cusca.dendodist(children, dist)
    if all(value == 0 for value in distance):
        # If all distances are zero, then use uniform distance
        distance = np.arange(len(distance))

    # The number of observations contained in each cluster level
    no_of_observations = np.arange(2, children.shape[0]+2)
    # Create linkage matrix and then plot the dendrogram
    linkage_matrix = np.column_stack([children, distance, no_of_observations]).astype(float)
    # Plot the corresponding dendrogram
    R = dendrogram(linkage_matrix, color_threshold=0, orientation='left', **kwargs)
    # [label.set_rotation(90) for label in plt.gca().get_xticklabels()]
    order_of_rows = R['ivl']
    # print(order_of_columns)
    plt.gca().get_xaxis().set_visible(False)

    return list(reversed(order_of_rows))

def my_affinity_generic(M, metric):
    return np.array([np.array([metric(a, b) for a in M])for b in M])

def my_affinity_i(M):
    return np.array([[cusca.information_coefficient_dist(a, b) for a in M]for b in M])

def my_affinity_ai(M):
    return np.array([[cusca.absolute_information_coefficient_dist(a, b) for a in M]for b in M])

def my_affinity_p(M):
    return np.array([[cusca.custom_pearson_dist(a, b) for a in M]for b in M])

def my_affinity_s(M):
    return np.array([[cusca.custom_spearman_dist(a, b) for a in M]for b in M])

def my_affinity_k(M):
    return np.array([[cusca.custom_kendall_tau_dist(a, b) for a in M]for b in M])

def my_affinity_ap(M):
    return np.array([[cusca.absolute_pearson_dist(a, b) for a in M]for b in M])

def my_affinity_u(M):
    return np.array([[cusca.uncentered_pearson_dist(a, b) for a in M]for b in M])

def my_affinity_au(M):
    return np.array([[cusca.absolute_uncentered_pearson_dist(a, b) for a in M]for b in M])

def my_affinity_l1(M):
    return np.array([[cusca.custom_manhattan_dist(a, b) for a in M]for b in M])

def my_affinity_l2(M):
    return np.array([[cusca.custom_euclidean_dist(a, b) for a in M]for b in M])

def my_affinity_m(M):
    return np.array([[cusca.custom_manhattan_dist(a, b) for a in M]for b in M])

def my_affinity_c(M):
    return np.array([[cusca.custom_cosine_dist(a, b) for a in M]for b in M])

def my_affinity_e(M):
    # global dist_matrix
    # dist_matrix = np.array([[cusca.mydist(a, b) for a in M]for b in M])
    # return dist_matrix
    return np.array([[cusca.custom_euclidean_dist(a, b) for a in M] for b in M])

def count_diff(x):
    count = 0
    compare = x[0]
    for i in x:
        if i != compare:
            count += 1
    return count

def count_mislabels(labels, true_labels):
    # 2017-08-17: I will make the assumption that clusters have only 2 values.
    # clusters = np.unique(true_labels)
    # mislabels = 0
    # for curr_clust in clusters:
    #     print("for label", curr_clust)
    #     print("\t", labels[(true_labels == curr_clust)])
    #     compare_to = mode(labels[(true_labels == curr_clust)])
    #     print("\tcompare to:", compare_to, "mislables: ", np.count_nonzero(labels[(true_labels == curr_clust)] != compare_to))
    #     mislabels += np.count_nonzero(labels[(true_labels == curr_clust)] != compare_to)

    set_a = labels[true_labels == 0]
    set_b = labels[true_labels == 1]

    if len(set_a) <= len(set_b):
        shorter = set_a
        longer = set_b
    else:
        shorter = set_b
        longer = set_a

    long_mode = mode(longer)  # this what the label of the longer cluster should be.
    short_mode = 1 if long_mode == 0 else 0  # Choose the other value for the label of the shorter cluster

    # start with the longer vector:
    # print("The long set is", longer, "it has", np.count_nonzero(longer != long_mode), 'mislabels.')
    # print("The short set is", shorter, "it has", np.count_nonzero(shorter != short_mode), 'mislabels.')

    # np.count_nonzero(longer != long_mode) + np.count_nonzero(shorter != short_mode)

    return np.count_nonzero(longer != long_mode) + np.count_nonzero(shorter != short_mode)

def plot_heatmap(df, col_order, row_order, top=5, title_text='differentially expressed genes per phenotype'):
    if not(len(col_order), len(list(df))):
        exit("Number of columns in dataframe do not match the columns provided for ordering.")
    if not(len(row_order), len(df)):
        exit("Number of rows in dataframe do not match the columns provided for ordering.")
    # print(list(df), col_order)
    df = df[col_order]
    df = df.reindex(row_order)

    plt.clf()
    sns.heatmap(df.iloc[np.r_[0:top, -top:0], :], cmap='viridis')
    plt.yticks(rotation=0)
    plt.xticks(rotation=90)
    plt.title('Top {} {}'.format(top, title_text))
    plt.ylabel('Genes')
    plt.xlabel('Sample')
    plt.savefig('heatmap.png', dpi=300, bbox_inches="tight")

def parse_data(gct_name, row_normalization=False, col_normalization=False, row_centering=None, col_centering=None):
    # f = open(gct_name)
    # f.readline()
    # size = f.readline().strip('\n').split('\t')

    data_df = pd.read_csv(gct_name, sep='\t', skiprows=2)
    # print(size)
    # print(list(data_df))
    # exit(data_df.shape)

    if 'Name' not in list(data_df):
        data_df['Name'] = data_df.iloc[:, 0]
        data_df.drop(data_df.columns[0], axis=1, inplace=True)
    if 'Description' not in list(data_df):
        data_df['Description'] = data_df['Name']

    data_df.set_index(data_df['Name'], inplace=True)
    og_full_gct = data_df.copy()
    og_full_gct.drop(['Name'], axis=1, inplace=True)
    data_df.drop(['Name', 'Description'], axis=1, inplace=True)
    plot_labels = list(og_full_gct.drop(['Description'], axis=1, inplace=False))
    data = data_df.as_matrix()
    row_labels = data_df.index.values

    og_data = data.copy()

    if row_centering is not None:
        if row_centering == 'Mean':
            row_means = np.mean(data, axis=1)
            row_means_col_vec = row_means.reshape((data.shape[0], 1))
            data = data - row_means_col_vec
        if row_centering == 'Median':
            row_medians = np.median(data, axis=1)
            row_medians_col_vec = row_medians.reshape((data.shape[0], 1))
            data = data - row_medians_col_vec

    if row_normalization:
        row_norm = np.sum(data * data, axis=1)
        row_norm_col_vec = row_norm.reshape((data.shape[0], 1))
        data = data / np.sqrt(row_norm_col_vec)

    if col_centering is not None:
        if col_centering == 'Mean':
            col_means = np.mean(data, axis=0)
            data = data - col_means
        if col_centering == 'Median':
            col_medians = np.median(data, axis=0)
            data = data - col_medians

    if col_normalization:
        col_norm = np.sum(data*data, axis=0)
        data = data/np.sqrt(col_norm)

    # print(data_df)
    # print(data)
    new_data_df = pd.DataFrame(data=data, index=data_df.index, columns=list(data_df))
    # print(new_data_df)
    # print(og_full_gct)
    new_full_gct = new_data_df.copy()
    new_full_gct.insert(0, column='Description', value=og_full_gct['Description'])
    # print(new_full_gct)
    # exit()

    return og_data, data_df, data, new_data_df, plot_labels, row_labels, og_full_gct, new_full_gct

str2func = {
    'custom_euclidean': my_affinity_e,
    'uncentered_pearson': my_affinity_u,
    'absolute_uncentered_pearson': my_affinity_au,
    'information_coefficient': my_affinity_i,
    'pearson': my_affinity_p,
    'spearman': my_affinity_s,
    'kendall': my_affinity_k,
    'absolute_pearson': my_affinity_ap,
    'l1': 'l1',
    'l2': 'l2',
    'manhattan': 'manhattan',
    'cosine': 'cosine',
    'euclidean': 'euclidean',
}

pdist_dict = {
    'custom_euclidean': 'correlation',
    'uncentered_pearson': my_affinity_u,
    'absolute_uncentered_pearson': my_affinity_au,
    'information_coefficient': cusca.information_coefficient_dist,
    'pearson': 'correlation',
    'spearman': cusca.custom_spearman_dist,
    'kendall': cusca.custom_kendall_tau_dist,
    'absolute_pearson': cusca.absolute_pearson_dist,
    'l1': 'cityblock',
    'l2': 'l2',
    'manhattan': 'cityblock',
    'cosine': 'cosine',
    'euclidean': 'euclidean',
}

str2affinity_func = {
    'custom_euclidean': my_affinity_e,
    'uncentered_pearson': my_affinity_u,
    'absolute_uncentered_pearson': my_affinity_au,
    'information_coefficient': my_affinity_i,
    'pearson': my_affinity_p,
    'spearman': my_affinity_s,
    'kendall': my_affinity_k,
    'absolute_pearson': my_affinity_ap,
    'l1': my_affinity_l1,
    'l2': my_affinity_l2,
    'manhattan': my_affinity_m,
    'cosine': my_affinity_c,
    'euclidean': my_affinity_e,
}

str2dist = {
    'custom_euclidean': cusca.custom_euclidean_dist,
    'uncentered_pearson': cusca.uncentered_pearson_dist,
    'absolute_uncentered_pearson': cusca.absolute_uncentered_pearson_dist,
    'information_coefficient': cusca.information_coefficient_dist,
    'pearson': cusca.custom_pearson_dist,
    'spearman': cusca.custom_spearman_dist,
    'kendall': cusca.custom_kendall_tau_dist,
    'absolute_pearson': cusca.absolute_pearson_dist,
    'l1': cusca.custom_manhattan_dist,
    'l2': cusca.custom_euclidean_dist,
    'manhattan': cusca.custom_manhattan_dist,
    'cosine': cusca.custom_cosine_dist,
    'euclidean': cusca.custom_euclidean_dist,
}

str2similarity = {
    'custom_euclidean': cusca.custom_euclidean_sim,
    'uncentered_pearson': cusca.uncentered_pearson_corr,
    'absolute_uncentered_pearson': cusca.absolute_uncentered_pearson_corr,
    'information_coefficient': cusca.information_coefficient,
    'pearson': cusca.custom_pearson_corr,
    'spearman': cusca.custom_spearman_corr,
    'kendall': cusca.custom_kendall_tau_corr,
    'absolute_pearson': cusca.absolute_pearson_corr,
    'l1': cusca.custom_manhattan_sim,
    'l2': cusca.custom_euclidean_sim,
    'manhattan': cusca.custom_manhattan_sim,
    'cosine': cusca.custom_cosine_sim,
    # 'euclidean': pairwise.paired_euclidean_distances,
    'euclidean': cusca.custom_euclidean_sim,
    # 'euclidean': cusca.custom_euclidean_dist,
}

linkage_dic = {
    'Pairwise average-linkage': 'average',
    'Pairwise complete-linkage': 'complete',
    'Pairwise ward-linkage': 'ward',
    'average': 'average',
    'complete': 'complete',
    'ward': 'ward',
}

def make_tree(model, n_leaves=None, scipy=False):
    """
    Modified from:
    https://stackoverflow.com/questions/27386641/how-to-traverse-a-tree-from-sklearn-agglomerativeclustering
    import numpy as np
    from sklearn.cluster import AgglomerativeClustering
    import itertools

    X = np.concatenate([np.random.randn(3, 10), np.random.randn(2, 10) + 100])
    model = AgglomerativeClustering(linkage="average", affinity="cosine")
    model.fit(X)

    ii = itertools.count(X.shape[0])
    [{'node_id': next(ii), 'left': x[0], 'right':x[1]} for x in model.children_]

    ---

    You can also do dict(enumerate(model.children_, model.n_leaves_))
    which will give you a dictionary where the each key is the ID of a node
    and the value is the pair of IDs of its children. – user76284

    :param model:
    :return: a dictionary where the each key is the ID of a node and the value is the pair of IDs of its children.
    """
    # ii = itertools.count(data.shape[0])  # Setting the counter at the number of leaves.
    # tree = [{'node_id': next(ii), 'left': x[0], 'right':x[1]} for x in model.children_]
    # print(tree)
    # return tree

    if scipy:
        return dict(enumerate(np.delete(model, [2, 3], 1).astype(int), n_leaves))

    else:

        return dict(enumerate(model.children_, model.n_leaves_))
    # return dict(enumerate(model.children_, 1))

def make_cdt(data, order_of_columns, order_of_rows, name='test.cdt', atr_companion=True, gtr_companion=False):
    # TODO: if order_of_columns == None, then do arange(len(list(data)))
    # TODO: if order_of_rows == None, then do arange(len(list(data)))
    # exit(data.to_csv())
    data.index.name = "ID"
    data.rename(columns={'Description': 'Name'}, inplace=True)

    temp = np.ones(len(data))
    data.insert(loc=1, column='GWEIGHT', value=temp)  # adding an extra column

    # These three lines add a row
    data.loc['EWEIGHT'] = list(np.ones(len(list(data))))
    newIndex = ['EWEIGHT'] + [ind for ind in data.index if ind != 'EWEIGHT']
    data = data.reindex(index=newIndex)

    if atr_companion:
        new_AID = ['', '']
        for element in range(len(order_of_columns)):
            temp = 'ARRY'+str(element)+'X'
            new_AID.append(temp)

        data.loc['AID'] = new_AID
        newIndex = ['AID'] + [ind for ind in data.index if ind != 'AID']
        data = data.reindex(index=newIndex)
        data = data[['Name', 'GWEIGHT']+order_of_columns]
    if gtr_companion:
        new_GID = ['']
        if atr_companion:
            new_GID = ['AID', 'EWEIGHT']  # This is to make sure we fit the CDT format
        # for element in np.sort(np.unique(GID)):
            # if 'NODE' in element:
            #     # print(element, 'GTR delete')
            #     pass
            # else:
            #     new_GID.append(element)
        for element in range(len(order_of_rows)):
            temp = 'GENE' + str(element) + 'X'
            new_GID.append(temp)

        data.insert(loc=0, column='GID', value=new_GID)  # adding an extra column
        data.insert(loc=0, column=data.index.name, value=data.index)  # Making the index a column

        # reorder to match dendogram
        temp = ['AID', 'EWEIGHT'] + order_of_rows
        data = data.loc[temp]
        # print(list(data.index))
        # print(data['GID'])
        # print(data['Name'])

        # Making the 'GID' the index -- for printing purposes
        data.index = data['GID']
        data.index.name = 'GID'
        data.drop(['GID'], axis=1, inplace=True)
        # print(list(data.index))

    # The first three lines need to be written separately due to a quirk in the CDT file format:

    # print(data.to_csv(sep='\t', index=True, header=True))
    f = open(name, 'w')
    f.write(data.to_csv(sep='\t', index=True, header=True))
    # f.write(data.to_csv(sep='\t', index=True, header=True))
    f.close()
    # pd.options.display.float_format = '{:3.3f}'.format
    data = data.round(2)
    # print(data.to_csv())
    # exit()
    # exit(data.to_csv(sep=' ', index=True, header=True, float_format='2',))
    return

def make_atr(col_tree_dic, data, dist, clustering_method='average', file_name='test.atr'):
    max_val = len(col_tree_dic)
    # AID = []

    # compute distances
    distance_dic = {}
    for node, children in col_tree_dic.items():
        val = centroid_distances(children[0], children[1], tree=col_tree_dic, data=data, axis=1,
                                 distance=dist, clustering_method=clustering_method)
        # print("Value is", val)
        distance_dic[node] = val

    # if dist == cusca.custom_euclidean_sim:
    #     print("Euclidean distance is especial, normalizing using this scheme:")
    #     low_norm = min(distance_dic.values())
    #     high_norm = max(distance_dic.values())
    #     for key in distance_dic.keys():
    #         # distance -= norm
    #         # distance_dic[key] = distance_dic[key]/high_norm
    #         # distance_dic[key] = (distance_dic[key]-low_norm)/high_norm
    #         # distance_dic[key] = distance_dic[key]/high_norm
    #         # distance_dic[key] = ((1/distance_dic[key])-high_norm)/low_norm
    #         print(distance_dic[key])

    f = open(file_name, 'w')
    for node, children in col_tree_dic.items():
        elements = [translate_tree(node, max_val, 'atr'), translate_tree(children[0], max_val, 'atr'),
                    translate_tree(children[1], max_val, 'atr'),
                    "{num:.{width}f}".format(num=distance_dic[node], width=SIGNIFICANT_DIGITS)]
        # print('\t', '\t'.join(elements))
        # AID.append(translate_tree(children[0], max_val, 'atr'))
        # AID.append(translate_tree(children[1], max_val, 'atr'))
        f.write('\t'.join(elements) + '\n')
        # print('\t'.join(elements) + '\n')
    f.close()

    return

def make_gtr(row_tree_dic, data, dist, clustering_method='average',  file_name='test.gtr'):
    max_val = len(row_tree_dic)
    # GID = []

    # compute distances
    distance_dic = {}
    for node, children in row_tree_dic.items():
        val = centroid_distances(children[0], children[1], tree=row_tree_dic, data=data, axis=0,
                                 distance=dist, clustering_method=clustering_method)
        distance_dic[node] = val

    f = open(file_name, 'w')
    for node, children in row_tree_dic.items():

        elements = [translate_tree(node, max_val, 'gtr'), translate_tree(children[0], max_val, 'gtr'),
                    translate_tree(children[1], max_val, 'gtr'),
                    "{num:.{width}f}".format(num=distance_dic[node], width=SIGNIFICANT_DIGITS)]
        # GID.append(translate_tree(children[0], max_val, 'gtr'))
        # GID.append(translate_tree(children[1], max_val, 'gtr'))
        f.write('\t'.join(elements)+'\n')
        # val -= 1
    f.close()

    return

def translate_tree(what, length, g_or_a):
    if 'a' in g_or_a:
        if what <= length:
            translation = 'ARRY'+str(what)+'X'
        else:
            translation = 'NODE' + str(what-length) + 'X'
    elif 'g' in g_or_a:
        if what <= length:
            translation = 'GENE'+str(what)+'X'
        else:
            translation = 'NODE' + str(what-length) + 'X'
    else:
        translation = []
        print('This function does not support g_or_a=', g_or_a)
    return translation

# def get_children_recursively(k, model, node_dict, leaf_count, n_samples, data, verbose=False, left=None, right=None):
#     # print(k)
#     i, j = model.children_[k]
#
#     if k in node_dict:
#         return node_dict[k]['children']
#
#     if i < leaf_count:
#         # print("i if")
#         left = [i]
#     else:
#         # print("i else")
#         # read the AgglomerativeClustering doc. to see why I select i-n_samples
#         left, node_dict = get_children_recursively(i - n_samples, model, node_dict,
#                                                    leaf_count, n_samples, data, verbose, left, right)
#
#     if j < leaf_count:
#         # print("j if")
#         right = [j]
#     else:
#         # print("j else")
#         right, node_dict = get_children_recursively(j - n_samples, model, node_dict,
#                                                     leaf_count, n_samples, data, verbose, left, right)
#
#     if verbose:
#         print(k, i, j, left, right)
#     temp = map(lambda ii: data[ii], left)
#     left_pos = np.mean(list(temp), axis=0)
#     temp = map(lambda ii: data[ii], right)
#     right_pos = np.mean(list(temp), axis=0)
#
#     # this assumes that agg_cluster used euclidean distances
#     dist = metrics.pairwise_distances([left_pos, right_pos], metric='euclidean')[0, 1]
#
#     all_children = [x for y in [left, right] for x in y]
#     pos = np.mean(list(map(lambda ii: data[ii], all_children)), axis=0)
#
#     # store the results to speed up any additional or recursive evaluations
#     node_dict[k] = {'top_child': [i, j], 'children': all_children, 'pos': pos, 'dist': dist,
#                     'node_i': k + n_samples}
#     return all_children, node_dict

# def recursive_atr

def get_children(tree, leaves_are_self_children=False):
    # this is a recursive function
    expanded_tree = {}
    for node in range(max(tree.keys())):
        if node <= len(tree):
            if leaves_are_self_children:
                expanded_tree[node] = [node]
            else:
                expanded_tree[node] = []

        else:
            # expanded_tree[node] = list_children_single_node(node, tree)
            expanded_tree[node] = list_children_single_node(node, tree, leaves_are_self_children)

    return expanded_tree

def list_children_single_node(node, tree, leaves_are_self_children=False, only_leaves_are_children=True):
    # children = []
    if node <= len(tree):
        if leaves_are_self_children:
            children = [node]
        else:
            children = []

    else:
        children = list(tree[node])

        # Check each child, and add their children to the list
        for child in children:
            if child <= len(tree):
                pass
            else:
                children += list_children_single_node(child, tree, only_leaves_are_children=True)
    if only_leaves_are_children:
        # print(sorted(np.unique(i for i in children if i <= len(tree))))
        # print()
        return [i for i in sorted(np.unique(children))if i <= len(tree)]
    else:
        return sorted(np.unique(children))

def centroid_distances(node_a, node_b, tree, data, axis=0, distance=cusca.mydist, clustering_method='average'):
    if axis == 0:
        pass
    elif axis == 1:
        data = np.transpose(data)
    else:
        exit("Variable 'data' does not have that many axises (╯°□°)╯︵ ┻━┻")

    children_of_a = list_children_single_node(node_a, tree=tree, leaves_are_self_children=True)
    children_of_b = list_children_single_node(node_b, tree=tree, leaves_are_self_children=True)

    # if distance == cusca.custom_euclidean_sim:
    #     print("Euclidean distance is especial, normalizing using this scheme:")
    #     distance = cusca.custom_euclidean_dist

    distances_list = []
    if clustering_method == 'average':
        for pair in itertools.product(data[children_of_a], data[children_of_b]):
            distances_list.append(distance(pair[0], pair[1]))
        return np.average(distances_list)
    elif clustering_method == 'complete':
        for pair in itertools.product(data[children_of_a], data[children_of_b]):
            distances_list.append(distance(pair[0], pair[1]))
        return np.min(distances_list)
    else:
        exit("Ony 'average' and 'complete' clustering methods are accepted at the moment (>_<)")

def euclidian_similarity(x, y):
    dist = cusca.mydist(x, y)
    # return 1/(1+dist)
    return 1/(np.exp(dist))

def better_dendodist(children, distance, tree, data, axis):
    distances_list = []
    for pair in children:
        distances_list.append(centroid_distances(pair[0], pair[1], tree, data, axis, distance=distance))
    return distances_list

def two_plot_2_dendrogram(Z, num_clust, no_plot=True):
    # plt.clf()
    threshold = Z[-num_clust + 1, 2]
    R = dendrogram(Z, no_labels=True, color_threshold=threshold, no_plot=no_plot)
    order_of_columns = R['leaves']

    # plt.show()
    # plt.savefig('WHAT_ARE_THOOOOSE.png')
    return order_of_columns, R

ext: py

max_stars_repo_name: genepattern/HierarchicalClustering

max_issues_repo_name: genepattern/HierarchicalClustering

max_forks_repo_name: genepattern/HierarchicalClustering

reason: import numpy,from scipy


text:
import numpy as np

class BicycleModel(object):
    """
    Class representing the state of a vehicle
    : param state : (float) numpy array [x-coordinate, y-coordinate, yaw-angle].T
    """
    # state indices
    xidx, yidx, thetaidx = 0, 1, 2
    # control max vals
    wheel_base = .7
    near_zero = 1e-05
    def __init__(self, initial_state, goal, v_max=3, omega_max=5):

        self.state_history = [initial_state]
        self.control_history = []
        self.state = initial_state
        self._goal = goal

        #Max Values
        self.v_max = v_max
        self.v_min = -v_max
        self.omega_max = omega_max
        self.omega_min = -omega_max

        self.A = self.system_matrix()
        self.B = self.control_matrix(initial_state[self.thetaidx])

    def update(self, state, control):
        self.state = state
        self.state_history.append(self.state)
        self.control_history.append(control)

    def move(self, controls, dT):

        self.state += self.get_throttle(controls, dT)
        self.state_history.append(self.state)
        return self.get_throttle(controls, dT)

    def get_throttle(self, controls, dT):
        return dT*np.matmul(self.B, controls)

    @property
    def goal(self):
        return self._goal

    @goal.setter
    def goal(self, destination):
        self._goal = destination

    def system_matrix(self):

        return np.array([[1, self.near_zero, self.near_zero],
                        [self.near_zero, 1, self.near_zero],
                        [self.near_zero, self.near_zero, 1]], dtype=np.float)

    @staticmethod
    def control_matrix(theta):
        return np.array([[np.cos(theta), 0],
                         [np.sin(theta), 0],
                        [0, 1]], dtype=np.float)

reason: import numpy


text:
import cv2 as cv
import numpy as np
import time

path_to_dataset = './data'
detector = cv.FastFeatureDetector_create()
associate_file = open(path_to_dataset + '/associate.txt')
line = associate_file.readline()
time_rgb, rgb_file, time_depth, depth_file = line.rstrip('\n').split(' ')

depth = cv.imread(path_to_dataset + "/" + depth_file, -1)
pre_color = cv.imread(path_to_dataset + "/" + rgb_file)
pre_kps = np.float32([kp.pt for kp in detector.detect(pre_color)])
index = 0

while True:
    line = associate_file.readline()
    if not line:
        break
    time_rgb, rgb_file, time_depth, depth_file = line.rstrip('\n').split(' ')
    color = cv.imread(path_to_dataset + "/" + rgb_file)
    depth = cv.imread(path_to_dataset + "/" + depth_file, -1)
    if color is None or depth is None:
        continue
    tic = time.time()
    next_kps, status, err = cv.calcOpticalFlowPyrLK(pre_color, color, pre_kps, None)
    toc = time.time()
    print('LK Flow use time：', toc - tic, ' seconds.')
    next_kps = next_kps[np.where(status != 0)[0], :]
    if len(next_kps) == 0:
        print('all keypoints are lost.')
        break
    print('tracked keypoints: ', next_kps.shape[0])
    img_show = np.copy(color)
    for kp in next_kps:
        cv.circle(img_show, (kp[0], kp[1]), 5, (0, 240, 0))
        cv.imshow("corners", img_show)
    cv.waitKey(0)
    pre_color = color
    pre_kps = next_kps
    # if index % 50 == 0 or len(pre_kps) == 0:
    #     sorted_kps = detector.detect(color)
    #     sorted_kps.sort(key=lambda x: x.response)
    #     pre_kps = np.float32([kp.pt for kp in sorted_kps])
    #     pre_kps = pre_kps[:min(pre_kps.shape[0], 500), :]
    index += 1


text:
import scipy.stats as stats
import seaborn as sns

class GammaDist():
    def __init__(self, **kwargs):
        self.__data_initialization(**kwargs)
        self.__generate_dataset()

    def __data_initialization(self, **kwargs):
        self.loc = kwargs["loc"]
        self.alpha = kwargs["alpha"]
        self.beta = kwargs["beta"]
        self.size = kwargs["size"]
        self.tetha = 1 / self.beta
        # self.st_range = kwargs["st_range"]

    def get_size(self):
        return self.size

    def __generate_dataset(self):
        self.rand_gamma = stats.gamma.rvs(self.alpha, scale=self.tetha, size=self.size)
        # self.rand_exp = np.random.exponential(self.beta, self.size)

    def __update_alpha(self, new_alpha):
        self.alpha = new_alpha

    def regenerate_dataset(self, new_alpha=None):
        if new_alpha is not None:
            self.__update_alpha(new_alpha)
        self.__generate_dataset()

    def get_dataset(self):
        return self.rand_gamma

    def get_alpha(self):
        return self.alpha

    def get_beta(self):
        return self.beta

    def plot_dataset(self):
        ax = sns.distplot(self.rand_gamma,
                          kde=True,
                          bins=self.size, # number of sliced data (number of x in plot)
                          color='black',
                          # color='skyblue',
                          hist_kws={"linewidth": 15, 'alpha': 1})
        ax.set(xlabel='Gamma Distribution', ylabel='Frequency')

max_stars_repo_name: ardihikaru/mlsp

max_issues_repo_name: ardihikaru/mlsp

max_forks_repo_name: ardihikaru/mlsp

reason: import scipy


text:
import itertools

import numpy as np

bytearray_lengths = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
                     20, 30, 32, 40, 50, 60, 64, 70, 80, 90, 100, 128]
byte8array_lengths = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]

basic_types = [
    'str4',
    'str8',

    'uint8',
    'uint16',
    'uint32',
    'uint64',

    'int8',
    'int16',
    'int32',
    'int64',

    'float32',
    'float64',
]

basic_cpp_types = {
    'str4' : 'std::array<char, 4>',
    'str8' : 'std::array<char, 8>',
    'str16' : 'std::array<char, 16>',
    'str32' : 'std::array<char, 32>',

    'int8' : 'int8_t',
    'int16' : 'int16_t',
    'int32' : 'int32_t',
    'int64' : 'int64_t',

    'uint8' : 'uint8_t',
    'uint16' : 'uint16_t',
    'uint32' : 'uint32_t',
    'uint64' : 'uint64_t',

    'float32' : 'float',
    'float64' : 'double',
}

basic_np_types = {
    'str4' : 'U4',
    'str8' : 'U8',
    'str16' : 'U16',
    'str32' : 'U32',

    'int8' : 'i1',
    'int16' : 'i2',
    'int32' : 'i4',
    'int64' : 'i8',

    'uint8' : 'u1',
    'uint16' : 'u2',
    'uint32' : 'u4',
    'uint64' : 'u8',

    'float32' : 'f4',
    'float64' : 'f8',
}

# Restricted list of keys and values to have a fast compilation

key_types = ['uint64']

set_key_types = []

value_types = ['uint8']

assert all([type_ in key_types for type_ in set_key_types])
assert all([type_ in basic_np_types for type_ in key_types])
assert all([type_ in basic_cpp_types for type_ in key_types])

cpp_types = {
    **{basic_type : basic_cpp_types[basic_type] for basic_type in basic_types},

    **{f'bytearray{i}' : f'bytearray{i}' for i in bytearray_lengths},

    **{f'byte8array{i}' : f'byte8array{i}' for i in byte8array_lengths},
}

np_types = {
    **{basic_type : basic_np_types[basic_type] for basic_type in basic_types},

    **{f'bytearray{i}' : {'names' : ['bytearray'], 'formats' : [f'{i}u1']} for i in bytearray_lengths},

    **{f'byte8array{i}' : {'names' : ['bytearray'], 'formats' : [f'{i}u8']} for i in byte8array_lengths},
}

generic_types = {
    **{f'bytearray{i}' : f'using bytearray{i} = bytearray<uint8_t, {i}>;' for i in bytearray_lengths},

    **{f'byte8array{i}' : f'using byte8array{i} = bytearray<uint64_t, {i}>;' for i in byte8array_lengths},
}

def create_dict_class_name(key_type, value_type):
    return f'Dict_{key_type}_{value_type}'

def create_set_class_name(key_type):
    return f'Set_{key_type}'

def write_dict_types_dict(getpy_file, key_type, value_type):
    class_name = create_dict_class_name(key_type, value_type)
    getpy_file.write(f"    (types['{key_type}'], types['{value_type}']) : _gp.{class_name},\n")

def write_set_types_dict(getpy_file, key_type):
    class_name = create_set_class_name(key_type)
    getpy_file.write(f"    (types['{key_type}']) : _gp.{class_name},\n")

def write_generic_types(getpy_file, type_):
    getpy_file.write(f'{generic_types[type_]}\n')

def write_numpy_dtype(getpy_file, type_):
    if type_.startswith('bytearray') or type_.startswith('byte8array'):
        getpy_file.write(f'    PYBIND11_NUMPY_DTYPE( {cpp_types[type_]}, bytearray );\n')

    else:
        pass

def write_declare_dict(getpy_file, key_type, value_type):
    class_name = create_dict_class_name(key_type, value_type)

    if 'float' in value_type:
        getpy_file.write(f'    declare_dict_without_bitwise_operations<{cpp_types[key_type]}, {cpp_types[value_type]}>(m, "{class_name}");\n')
    else:
        getpy_file.write(f'    declare_dict<{cpp_types[key_type]}, {cpp_types[value_type]}>(m, "{class_name}");\n')

def write_declare_set(getpy_file, key_type):
    class_name = create_set_class_name(key_type)

    getpy_file.write(f'    declare_set<{cpp_types[key_type]}>(m, "{class_name}");\n')

def write_types_dict(getpy_file, type_):
    if isinstance(np_types[type_], dict):
        getpy_file.write(f"    '{type_}' : np.dtype({np_types[type_]}),\n")
    else:
        getpy_file.write(f"    '{type_}' : np.dtype('{np_types[type_]}'),\n")

def write_getpy_cpp(key_types, value_types):
    with open('src/getpy_types.cpp', 'w') as getpy_file:
        getpy_file.write("""\
#include "getpy.cpp"

#include <pybind11/pybind11.h>

""")
        all_types = value_types + [k for k in key_types if k not in value_types]
        for type_ in all_types:
            if type_ in generic_types:
                write_generic_types(getpy_file, type_)
            else:
                pass

        getpy_file.write("""\

PYBIND11_MODULE(_getpy, m) {
    m.doc() = "A Fast and Memory Efficient Hash Map for Python";

""")

        for type_ in all_types:
            if type_ in generic_types:
                write_numpy_dtype(getpy_file, type_)
            else:
                pass

        getpy_file.write('\n')

        for key_type, value_type in itertools.product(key_types, value_types):
            write_declare_dict(getpy_file, key_type, value_type)

        getpy_file.write('\n')

        for key_type in set_key_types:
            write_declare_set(getpy_file, key_type)

        getpy_file.write("""\
}
""")

def write_getpy_py(key_types, value_types):
    with open('getpy/getpy_types.py', 'w') as getpy_file:
        getpy_file.write("""\
import numpy as np

import _getpy as _gp

types = {
""")
        all_types = value_types + [k for k in key_types if k not in value_types]
        for _type in all_types:
            write_types_dict(getpy_file, _type)

        getpy_file.write("""\
}
""")

        getpy_file.write("""\

dict_types = {
""")

        for key_type, value_type in itertools.product(key_types, value_types):
            write_dict_types_dict(getpy_file, key_type, value_type)

        getpy_file.write("""\
}
""")

        getpy_file.write("""\

set_types = {
""")

        for key_type in set_key_types:
            write_set_types_dict(getpy_file, key_type)

        getpy_file.write("""\
}
""")

if __name__ == '__main__':
    write_getpy_cpp(key_types, value_types)
    write_getpy_py(key_types, value_types)

max_stars_repo_name: ksob/getpy

max_issues_repo_name: ksob/getpy

max_forks_repo_name: ksob/getpy


text:
# coding: utf-8

#加载内置包
import requests
import json
import time
import sys
import csv
import json
import os
import random
#加载第三方包
import pandas
from selenium import webdriver
from requests.exceptions import RequestException
from selenium.common.exceptions import WebDriverException
import cv2
from PIL import Image as Im
from selenium.webdriver import ActionChains
import numpy as np
from selenium.webdriver.chrome.options import Options
#加载自己编写的文件
import qqlist
import settings
import transCoordinateSystem
from get_grid import create_yichuxing_data

SYSTEM = 1 # 0: WINDOWS 1: LINUX

chrome_options = Options()
#chrome_options.add_argument('--no-sandbox')
#chrome_options.add_argument('--disable-dev-shm-usage')
#chrome_options.add_argument('--headless')
chrome_options.add_argument("enable-automation")
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--dns-prefetch-disable")
chrome_options.add_argument("--disable-gpu")
#chrome_options.setPageLoadStrategy(PageLoadStrategy.NORMAL);

class CookieException(Exception):
    def __init__(self):
        Exception.__init__(self)

class easygospider():
    #初始化基本量
    def __init__(self):
        self.qq_number_list = []
        self.cookie = None
        #输入文件的名称
        self.input = settings.xy_name
        #文件保存的路径
        self.filepath = settings.filepath
        self.filename = settings.filename
        self.qq_number_list = qqlist.qq_list
        self.captcha_path = settings.captcha_path

        # self.cookie = self.get_cookie()
        self.init_existed_files()

    def init_existed_files(self):
        self.existed_files = []
        for dir in os.listdir(self.filepath):
            filePath = os.path.join(self.filepath, dir)  # 得到文件夹和文件的完整路径
            if os.path.isdir(filePath):
               self.existed_files.append(dir)
        print(self.existed_files)

    # 初始化用于爬虫的网格，形成url
    def initial_paramslist(self):
        """
        :return: List[]
        """
        create_yichuxing_data()

        #filename文件中需要存储的为Wgs84坐标系下渔网为2.5Km的中心点的坐标，格式如data.txt
        #读取渔网中心点
        center = []
        with open(self.input, 'r', encoding='utf-8') as f:
            for item in f.readlines()[1:]:
                center.append(tuple(item.strip().split(",")[-2:]))
        #生成四至范围（按照2.6km范围生成，防止有遗漏点，如有重复，最后去重的时候处理）
        round = []
        for item in center:
            lng, lat = item
            lng, lat = float(lng), float(lat)
            round.append([lng - 0.5 * settings.lng_delta,
                          lng + 0.5 * settings.lng_delta,
                          lat - 0.5 * settings.lat_delta,
                          lat + 0.5 * settings.lat_delta])
        #生成待抓取的params
        paramslist = []
        for item in round:
            lng_min, lng_max, lat_min, lat_max = item
            #lng_min, lat_min = transCoordinateSystem.wgs84_to_gcj02(lng_min, lat_min)
            #lng_max, lat_max = transCoordinateSystem.wgs84_to_gcj02(lng_max, lat_max)
            params = {"lng_min": lng_min,
                      "lat_max": lat_max,
                      "lng_max": lng_max,
                      "lat_min": lat_min,
                      "level": 16,
                      "city": "杭州",
                      "lat": "undefined",
                      "lng": "undefined",
                      "_token": ""}
            paramslist.append(params)
        return paramslist

    def get_distance(self, small_url, big_url):
        """
        :param small_url: 是小图的路径(本地)
        :param big_url: 是大图的路径(本地)
        :return: 计算出移动的距离
        """
        # 引用上面的图片下载
        otemp = self.pic_download(small_url, 'small')
        time.sleep(2)
        # 引用上面的图片下载
        oblk = self.pic_download(big_url, 'big')
        # 计算拼图还原距离
        target = cv2.imread(otemp, 0)
        template = cv2.imread(oblk, 0)
        w, h = target.shape[::-1]
        temp = self.captcha_path + 'temp.jpg'
        targ = self.captcha_path + 'targ.jpg'
        cv2.imwrite(temp, template)
        cv2.imwrite(targ, target)
        target = cv2.imread(targ)
        target = cv2.cvtColor(target, cv2.COLOR_BGR2GRAY)
        target = abs(255 - target)
        cv2.imwrite(targ, target)
        target = cv2.imread(targ)
        template = cv2.imread(temp)
        result = cv2.matchTemplate(target, template, cv2.TM_CCOEFF_NORMED)
        x, y = np.unravel_index(result.argmax(), result.shape)
        # 缺口位置
        print((y, x, y + w, x + h))
        # 调用PIL Image 做测试
        image = Im.open(oblk)
        xy = (y + 20, x + 20, y + w - 20, x + h - 20)
        # 切割
        imagecrop = image.crop(xy)
        # 保存切割的缺口
        imagecrop.save(self.captcha_path + "new_image.jpg")
        return y

    def pic_download(self, url, pic_type):
        """
        :param url:  是图片的链接
        :param pic_type: 区分左侧小拼图和大图,大图传big,小图传small
        :return: 图片下载到本地,返回图片保存路径
        """
        path = self.captcha_path + pic_type + '.png'
        try:
            if not os.path.exists(self.captcha_path):
                os.makedirs(self.captcha_path)
            if os.path.exists(path):
                os.remove(path)
            r = requests.get(url)
            r.raise_for_status()
            with open(path, "wb") as f:  # 开始写文件，wb代表写二进制文件
                f.write(r.content)
                # print(f.name)
            print("{}图片下载完成".format(pic_type))
            return f.name

        except Exception as e:
            print("获取失败!" + str(e))
            self.pic_download(url, pic_type)

    def remove_captcha_old(self, driver):
        # 切换最初的frame中
        # driver.switch_to.default_content()
        # 切换frame
        # driver.switch_to.frame('login_frame')
        # 切换带有刷新按钮的frame
        driver.switch_to.frame(driver.find_element_by_xpath('//*[@id="newVcodeIframe"]/iframe'))

        # 点击刷新 id为e_reload
        driver.find_element_by_id('e_reload').click()
        # 获取图片链接
        big_url = driver.find_element_by_id('slideBg').get_attribute('src')
        small_url = driver.find_element_by_id('slideBlock').get_attribute('src')
        # 下载图片并计算拼图还原的距离
        y = self.get_distance(small_url, big_url)
        # 获取当前网页链接，用于判断拖动验证码后是否成功,如果拖动后地址没变则为失败
        slide_before_url = driver.current_url
        # 获取蓝色拖动按钮对象
        element = driver.find_element_by_id('tcaptcha_drag_button')
        # 计算distance（记住要用浮点型数字，不然计算后会取整）
        distance = y * (280.0 / 680.0) - 31
        print('distance:', distance)

        write_log(driver.page_source)
        for i in range(3):
            # 模拟人为拖动按钮
            has_gone_dist = 0
            remaining_dist = distance
            # remaining_dist += random.randint(-5, 5)
            # 按下鼠标左键
            ActionChains(driver).click_and_hold(element).perform()
            time.sleep(0.5)
            while remaining_dist > 0:
                ratio = remaining_dist / distance
                if ratio < 0.2:
                    # 开始阶段移动较慢
                    span = random.randint(5, 8)
                elif ratio > 0.8:
                    # 结束阶段移动较慢
                    span = random.randint(6, 8)
                else:
                    # 中间部分移动快
                    span = random.randint(10, 16)

                if span > remaining_dist:
                    span = remaining_dist
                ActionChains(driver).move_by_offset(span, random.randint(-2,2)).perform()
                remaining_dist -= span
                has_gone_dist += span
                print(remaining_dist, span)
                time.sleep(random.randint(5, 20) / 100)

            # print(remaining_dist+span)
            # ActionChains(driver).move_by_offset(remaining_dist, random.randint(-2,2)).perform()
            # time.sleep(random.randint(5, 20) / 100)
            ActionChains(driver).release(on_element=element).perform()
            time.sleep(1.0)

            try:
                # alert = driver.find_element_by_class_name('tcaptcha-title').text
                text = driver.find_element_by_id('guideText').text
                print(text)
                if text == '拖动下方滑块完成拼图':
                    break
                else:
                    distance += 5#random.randint(-5,5)
                # break
            except Exception as e:
                # print('滑块验证失败',e)
                distance += 5
                continue
            # if "拖动下方滑块完成拼图" in driver.page_source:
            #     break
            # else:
            #     print('滑块验证失败')
            #     distance += 5

        time.sleep(5)

    def remove_captcha(self, driver):
        # 切换带有刷新按钮的frame
        driver.switch_to.frame(driver.find_element_by_xpath('//*[@id="newVcodeIframe"]/iframe'))

        # 点击刷新 id为e_reload
        driver.find_element_by_id('e_reload').click()
        # 获取当前网页链接，用于判断拖动验证码后是否成功,如果拖动后地址没变则为失败
        slide_before_url = driver.current_url
        # 获取蓝色拖动按钮对象
        element = driver.find_element_by_id('tcaptcha_drag_button')

        distance = 170
        print('distance:', distance)

        write_log(driver.page_source)
        for i in range(3):
            # 模拟人为拖动按钮
            has_gone_dist = 0
            remaining_dist = distance
            # remaining_dist += random.randint(-5, 5)
            # 按下鼠标左键
            ActionChains(driver).click_and_hold(element).perform()
            time.sleep(0.5)
            while remaining_dist > 0:
                ratio = remaining_dist / distance
                if ratio < 0.2:
                    # 开始阶段移动较慢
                    span = random.randint(5, 8)
                elif ratio > 0.8:
                    # 结束阶段移动较慢
                    span = random.randint(6, 8)
                else:
                    # 中间部分移动快
                    span = random.randint(10, 16)

                if span > remaining_dist:
                    span = remaining_dist
                ActionChains(driver).move_by_offset(span, random.randint(-2,2)).perform()
                remaining_dist -= span
                has_gone_dist += span
                print(remaining_dist, span)
                time.sleep(random.randint(5, 20) / 100)

            # print(remaining_dist+span)
            # ActionChains(driver).move_by_offset(remaining_dist, random.randint(-2,2)).perform()
            # time.sleep(random.randint(5, 20) / 100)
            ActionChains(driver).release(on_element=element).perform()
            time.sleep(5.0)

            print(driver.current_url, slide_before_url)
            if driver.current_url == slide_before_url:
                distance -= random.randint(1,5)
            else:
                break
            # try:
                # # alert = driver.find_element_by_class_name('tcaptcha-title').text
                # text = driver.find_element_by_id('guideText').text
                # print(text)
                # if text == '拖动下方滑块完成拼图':
                #     break
                # else:
                #     distance -= 5#random.randint(-5,5)
                # break
            # except Exception as e:
            #     # print('滑块验证失败',e)
            #     distance -= 5
            #     continue
            # if "拖动下方滑块完成拼图" in driver.page_source:
            #     break
            # else:
            #     print('滑块验证失败')
            #     distance += 5

        # time.sleep(5)

    def get_cookie(self):
        for qq in self.qq_number_list:
            qq_num = qq[0]
            qq_passwd = qq[1]

            try:
                if SYSTEM == 0:
                    chrome_login = webdriver.Chrome(executable_path="chromedriver.exe",chrome_options=chrome_options)
                else:
                    chrome_login = webdriver.Chrome(executable_path="../chromedriver",chrome_options=chrome_options)
                # chrome_login.implicitly_wait(2)

                chrome_login.get("https://mail.qq.com/")
                print(1)
                time.sleep(1)
                # 切换frame login_frame是该登录窗口iframe的id
                chrome_login.switch_to.frame("login_frame")
                # 点击选择帐号密码登录
                # print(1.0)
                # print(chrome_login.page_source)
                # write_log(chrome_login.page_source)
                # chrome_login.get_screenshot_as_file("init.png")
                time.sleep(1)
                # chrome_login.execute_script("window.scrollTo(0, window.scrollY + 100)")
                # chrome_login.get_screenshot_as_file("full.png")
                # try:
                    # element = chrome_login.find_element_by_id("switcher_plogin")
                    # print(element, element.is_displayed(), element.is_enabled(), element.location)
                    # element.screenshot('./scr.png')
                # except WebDriverException as e:
                    # print('eee',e)
                # ActionChains(chrome_login).move_to_element(element).perform()
                # element.click()
                try:
                    chrome_login.find_element_by_id("switcher_plogin").click()
                except WebDriverException as e:
                    pass
                # time.sleep(1)
                # ActionChains(chrome_login).click(element).perform()
                # chrome_login.find_element_by_css_selector('#switcher_plogin').click()
                print(2)
                time.sleep(1)
                chrome_login.find_element_by_id("u").send_keys(qq_num)
                chrome_login.find_element_by_id("p").send_keys(qq_passwd)
                time.sleep(0.5)
                chrome_login.find_element_by_id("login_button").click()
                # 检查是否存在验证码
                print(3)
                time.sleep(2)
                if "安全验证" in chrome_login.page_source:
                    time.sleep(1)
                    write_log(chrome_login.page_source)
                    chrome_login.get_screenshot_as_file("screenshot/"+str(time.time())+"_verify.png")
                    if settings.CAPTCHA_RECOGNIZ:
                        # input('等待手动去除验证码')
                        # time.sleep(10)
                        self.remove_captcha(chrome_login)
                        # time.sleep(2)
                    else:
                        chrome_login.close()
                        continue
                time.sleep(1)
                chrome_login.get_screenshot_as_file("screenshot/"+str(time.time())+".png")
                write_log("AFTER VERIFY "+chrome_login.page_source)
                if "登录验证" in chrome_login.page_source:
                    write_log("登录验证 "+chrome_login.page_source)
                    chrome_login.find_element_by_id("login_button").click() #发送验证码
                    code = input('短信验证码')
                    try:
                        num = int(code)
                        chrome_login.find_element_by_id("u").send_keys(code)
                        chrome_login.get_screenshot_as_file("screenshot/"+str(time.time())+"_code.png")
                    except Exception as e:
                        print(e)
                # 获取cookie
                print(4)
                time.sleep(1)
                cookies = chrome_login.get_cookies()
                # print(cookies)
                write_log(str(qq_num)+" get cookie successful")
                # write_log(str(cookies))
                print(5)
                time.sleep(1)
                chrome_login.quit()
                user_cookie = {}
                for cookie in cookies:
                    user_cookie[cookie["name"]] = cookie["value"]
                print(user_cookie)
                write_log(str(user_cookie))

                self.cookie = user_cookie
                break
            except WebDriverException as e:
                print('Exception:',e)
                pass
            finally:
                try:
                    print('close')
                    chrome_login.close()
                except Exception as e:
                    print('Exception', e)
                    pass

    def spider(self,cookie,params):
        user_header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
            "Referer": "http://c.easygo.qq.com/eg_toc/map.html?origin=csfw"
        }
        url = "http://c.easygo.qq.com/api/egc/heatmapdata"
        # write_log(str(cookie))
        while True:
            try:
                r = requests.get(url, headers=user_header,
                                 cookies=cookie, params=params)
                # print(r.status_code)
                if r.status_code == 200 and r.text[2]!='c': # code=3:error
                    write_log(r.text)
                    return r.text
                else:
                    raise CookieException
            except RequestException:
                self.spider(cookie, params)

    def save(self,text,time_now,file_name):
        try:
            with open(file_name, 'r') as f:
                f.readline()
        except FileNotFoundError as e:
            with open(file_name, 'w', encoding='utf-8') as f:
                f.write('count,wgs_lng,wgs_lat,time\n')
        # 写入数据
        with open(file_name, "a", encoding="utf-8") as f:
            if text is None:
                return
            # print(text)
            node_list = json.loads(text)["data"]
            try:
                min_count = json.loads(text)['max_data']/40
                for i in node_list:
                    # 此处的算法在宜出行网页后台的js可以找到，文件路径是http://c.easygo.qq.com/eg_toc/js/map-55f0ea7694.bundle.js
                    i['count'] = i['count'] // min_count
                    gcj_lng = 1e-6 * (250.0 * i['grid_x'] + 125.0)
                    gcj_lat = 1e-6 * (250.0 * i['grid_y'] + 125.0)
                    lng, lat = transCoordinateSystem.gcj02_to_wgs84(gcj_lng, gcj_lat)
                    f.write(str(i['count']) + "," + str(lng) + "," + str(lat) + "," + time_now + "\n")
            except IndexError as e:
                pass
                # print("此区域没有点信息")
            except TypeError as e:
                # print(node_list)
                raise CookieException
    def remove_duplicate(self,filepath):
        # df = pandas.read_csv(filepath,sep=",")
        df = pandas.read_csv(filepath, sep=',')
        df = df.drop_duplicates()
        csv_name = filepath.replace(".txt", "去重结果.csv")
        df.to_csv(csv_name,index=False)

    def exec_once(self):
        # if self.cookie == None:
            # self.cookie = self.get_cookie()
        file_path = self.filepath + self.filename + '/' + self.filename

        while True:
            time_now = time.time()
            time_now_str = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time_now))
            write_log("此轮抓取开始")
            # cookie = self.get_cookie()
            i = 1
            params_list = self.initial_paramslist()
            for params in params_list:
                # print('params:',params)
                #这部分负责每个qq号码抓取的次数
                # if i % settings.fre == 0:
                    # cookie = self.get_cookie()
                while True:
                    try:
                        text = self.spider(self.cookie, params)
                        tmp_file_path = self.filepath + self.filename + '/'  + time_now_str
                        # self.save(text, time_now_str, file_name=self.filepath + self.filename + time_now_str + ".txt")
                        self.save(text, time_now_str, file_name=tmp_file_path + ".txt")
                        break
                    except CookieException as e:
                        self.get_cookie()

                view_bar(i, len(params_list))
                i += 1
            if os.path.exists(file_path + ".txt"):
                os.remove(file_path + ".txt")
            os.rename(tmp_file_path + ".txt", file_path + ".txt")

            if os.path.exists(tmp_file_path + ".txt"):
                os.remove(tmp_file_path + ".txt")

            write_log("写入json")
            # file_path = self.filepath + self.filename + time_now_str# + ".txt"
            csvfile = open(file_path+'.txt', 'r')
            jsonfile = open(file_path+'.json', 'w')
            reader = csv.DictReader(csvfile)
            out = json.dumps([row for row in reader])
            # print(out)
            jsonfile.write(out)

            # break
            return file_path

    def get_population_data(self, filename, left_top, right_top, left_down, right_down):
        # region config
        settings.city_bound_point_A = left_top
        settings.city_bound_point_B = right_top
        settings.city_bound_point_C = right_down
        settings.city_bound_point_D = left_down
        self.filename = settings.filename = filename
        self.input = settings.xy_name = settings.filepath + filename +'grid.txt'

        out_filepath = self.filepath + self.filename + '/' + self.filename
        out_dir = self.filepath + self.filename
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        if not os.path.exists(out_filepath+'.json'):
            if self.cookie == None:
                self.get_cookie()
            out_filepath = self.exec_once()

        data = []
        with open(out_filepath+'.json','r') as f:
            data = f.read()
        return data

def write_log(content):
    with open("log.log",'a',encoding='utf-8') as f:
        info = time.strftime("%Y-%m-%d %H:%M:%S ",time.localtime())+content+"\n"
        # sys.stdout.write(info)
        f.write(info)

def view_bar(num, total):
    rate = float(num) / float(total)
    rate_num = int(rate * 100)
    r = '\r[%s%s]%d%%' % ("="*(rate_num), " "*(100-rate_num), rate_num, )
    sys.stdout.write(r)

if __name__ == "__main__":
    app = easygospider()
    # app.exec()
    # app.get_cookie()


text:
import numpy as np
from GaussCondition import gauss_condition
def gauss_impute(model, X):
    """
    基于高斯分布进行数据重构
    Input:
        model:  包含属性mu和sigma的类
        X:       含缺失值的设计矩阵
    Output:
        Xc[i, j] = E[X[i, j]|D]
        V[i, j] = Var
    """
    n_samples, dim = X.shape
    Xc = X.copy()
    V = np.zeros((n_samples, dim))

    for i in range(n_samples):
        hid_index = np.where(np.isnan(X[i, :]))[0]
        if len(hid_index)==0: continue
        vis_index = np.where(~ np.isnan(X[i, :]))[0]
        vis_value = X[i, vis_index]
        mu_v, sigma_v = gauss_condition(model, vis_index, vis_value)
        Xc[i, hid_index] = mu_v
        V[i, hid_index] = np.diag(sigma_v)
    return Xc, V

max_stars_repo_name: xishansnow/MLAPP

max_issues_repo_name: xishansnow/MLAPP

max_forks_repo_name: xishansnow/MLAPP