text:
# -*- coding: utf-8 -*-
# @Time : 2020/7/6 15:56
# @Author : Zhongyi Hua
# @FileName: correct.py
# @Usage: correct annotation error using two gff result.
# @Note:
# @E-mail: njbxhzy@hotmail.com

import gffutils
import pandas as pd
import numpy as np
from Bio import SeqIO
from check import CheckCp, add2_rps12
import os

get_seq = lambda seq, start, end: seq.seq[start - 1:end]

def flatten(ls):
    def faction(lis):
        for i in lis:
            if isinstance(i, list):
                faction(i)
            else:
                flatten_list.append(i)

    flatten_list = []
    faction(ls)
    return flatten_list

class CorrectGff:
    gff_fields = ["seqid", "source", "type", "start", "end", "score", "strand", "phase", "attributes"]

    def __init__(self, gff_path_ge, gff_path_pga, seq_path, gff_new_path, seq_id, species_pre):
        self.species_pre = species_pre
        self.seq_id = seq_id
        self.seq = SeqIO.read(seq_path, 'fasta')
        self.gff = gffutils.create_db(gff_path_ge, ':memory:', merge_strategy='create_unique')
        self.gff_new_path = gff_new_path
        self.gff_pga = pd.read_table(gff_path_pga, comment='#', names=CorrectGff.gff_fields)
        # avoid duplicated index in geseq_dialect and pga_dialect
        self.gff_pga.index = self.gff_pga.index + 1000
        # correction will be conducted on this object
        self.gff_corrected = pd.read_table(gff_path_ge, comment='#', names=CorrectGff.gff_fields)
        # following objects would be used to record relationship
        self.geseq_dialect = pd.DataFrame()
        self.pga_dialect = pd.DataFrame()
        print('Start correct', os.path.basename(gff_path_ge))

    @staticmethod
    def get_parent(index, attributes):
        return index, {k.split('=')[0]: k.split('=')[1] for k in attributes.split(';')}['Parent']

    def create_geseq_dialect(self):
        # gene.id - locus - index - child_index(list)
        self.geseq_dialect['gene_id'] = self.gff_corrected.loc[self.gff_corrected['type'] == 'gene', 'attributes']. \
            apply(lambda x: x.split(';')[0].split('=')[1])
        self.geseq_dialect['index'] = self.gff_corrected.index.to_series()
        tmp_df = self.gff_corrected.loc[self.gff_corrected['type'] == 'CDS'].\
            apply(lambda x: self.get_parent(x.name, x['attributes']), axis=1, result_type="expand").\
            groupby(1).agg(list).\
            reset_index().\
            rename(columns={0: 'child_list', 1: 'gene_id'})
        try:
            tmp_df2 = self.gff_corrected.loc[self.gff_corrected['type'].isin(['tRNA', 'exon'])].\
                apply(lambda x: self.get_parent(x.name, x['attributes']), axis=1, result_type="expand")
            tmp_df2[1] = tmp_df2.apply(lambda x: str.replace(x[1], 'rna_', ''), axis=1)
            tmp_df2 = tmp_df2.\
                groupby(1).\
                agg(list).\
                reset_index().\
                rename(columns={0: 'child_list', 1: 'gene_id'})
            tmp_df = tmp_df.append(tmp_df2, ignore_index=True)
        except:
            pass
        self.geseq_dialect = self.geseq_dialect.merge(tmp_df, how='right')
        self.geseq_dialect = self.gff_corrected.loc[self.gff_corrected.index.isin(self.geseq_dialect['index']), 'attributes'].\
            apply(lambda x: {k.split('=')[0]: k.split('=')[1] for k in x.split(';')}['Name']).\
            reset_index().\
            rename(columns={'attributes': 'locus'}).\
            merge(self.geseq_dialect)

    def create_pga_dialect(self):
        # gene.id - index - locus - child_index(list)
        self.pga_dialect['gene_id'] = self.gff_pga.loc[self.gff_pga['type'] == 'gene', 'attributes']. \
            apply(lambda x: x.split(';')[0].split('=')[1])
        self.pga_dialect['index'] = self.gff_pga.index.to_series()
        tmp_df = self.gff_pga.loc[self.gff_pga['type'] == 'CDS'].\
            apply(lambda x: self.get_parent(x.name, x['attributes']), axis=1, result_type="expand").\
            groupby(1).agg(list).\
            reset_index().\
            rename(columns={0: 'child_list', 1: 'gene_id'})
        tmp_df2 = self.gff_pga.loc[self.gff_pga['type'].isin(['tRNA', 'exon'])].\
            apply(lambda x: self.get_parent(x.name, x['attributes']), axis=1, result_type="expand")
        tmp_df2[1] = tmp_df2.apply(lambda x: str.replace(x[1], 'rna_', ''), axis=1)
        tmp_df2 = tmp_df2.\
            groupby(1).\
            agg(list).\
            reset_index().\
            rename(columns={0: 'child_list', 1: 'gene_id'})
        tmp_df =tmp_df.append(tmp_df2, ignore_index=True)
        self.pga_dialect = self.pga_dialect.merge(tmp_df, how='right')
        self.pga_dialect = self.gff_pga.loc[self.gff_pga.index.isin(self.pga_dialect['index']), 'attributes'].\
            apply(lambda x: {k.split('=')[0]: k.split('=')[1] for k in x.split(';')}['Name']).\
            reset_index().\
            rename(columns={'attributes': 'locus'}).\
            merge(self.pga_dialect)

    def _correct_record(self, gene_feature):
        """
        correct a gene feature (with children)
        :param gene_feature:
        :return:
        """
        locus = gene_feature.attributes['Name'][0]
        drop_list = self.geseq_dialect.loc[self.geseq_dialect['gene_id'] == gene_feature.id, 'child_list'].to_list()
        drop_list.append(self.geseq_dialect.loc[self.geseq_dialect['gene_id'] == gene_feature.id, 'index'].to_list())
        drop_list = [k for x in drop_list for k in x]
        if len(self.pga_dialect.loc[self.pga_dialect['locus'] == locus]) > 1:
            tmp_dict = {}
            for _ in self.pga_dialect.loc[self.pga_dialect['locus'] == locus, 'index'].to_list():
                tmp_dict.update(
                    {self.gff_pga.loc[[_]].apply(lambda x: (x['start'] + x['end'])/2, axis=1).values[0]: _}
                )
            locus_idx = tmp_dict[
                min([_ for _ in tmp_dict.keys()], key=lambda x: abs((gene_feature.start + gene_feature.end)/2-x))
            ]
            gain_list = self.pga_dialect.loc[self.pga_dialect['index'] == locus_idx, 'child_list'].to_list()
            gain_list.append(self.pga_dialect.loc[self.pga_dialect['index'] == locus_idx, 'index'].to_list())
            gain_list = [k for x in gain_list for k in x]
        elif len(self.pga_dialect.loc[self.pga_dialect['locus'] == locus]) == 0:
            print('please check', locus, gene_feature.start, gene_feature.end, '(it is not in PGA)')
            return
        else:
            gain_list = self.pga_dialect.loc[self.pga_dialect['locus'] == locus, 'child_list'].to_list()
            gain_list.append(self.pga_dialect.loc[self.pga_dialect['locus'] == locus, 'index'].to_list())
            gain_list = [k for x in gain_list for k in x]
        self.gff_corrected.drop(drop_list, inplace=True)
        self.gff_corrected = self.gff_corrected.append(self.gff_pga.loc[gain_list])

    def _add_record(self, locus_ls):
        tmp_list = np.array(self.pga_dialect.loc[self.pga_dialect['locus'].isin(locus_ls), ['index', 'child_list']]).tolist()
        gain_list = flatten(tmp_list)
        self.gff_corrected = self.gff_corrected.append(self.gff_pga.loc[gain_list])

    def _add_pseudo(self, gene):
        self.gff_corrected.loc[self.gff_corrected['attributes'].str.startswith('ID='+gene.id), 'attributes'] += ';pseudo=true'

    def _find_start_codon(self, gene, _seq_combined):

        def _find_start(_seq, _count):
            _seq = _seq[3:]
            _count += 1
            if _seq == '' or _count > 9:
                return None
            try:
                _seq.translate(cds=True, table=11)
                return _count*3
            except:
                _find_start(_seq, _count)

        shift_num = _find_start(_seq_combined, 0)
        if shift_num is None:
            self._add_pseudo(gene)
            return print(gene.id, 'Maybe RNA-editing, but no alternative start codon find')
        else:
            print(gene.id, 'Mayebe RNA-editing, and a alternative start codon found')
            if gene.strand == '-':
                self.gff_corrected.loc[self.gff_corrected['attributes'].str.startswith('ID=' + gene.id), 'end'] \
                 = int(self.gff_corrected.loc[self.gff_corrected['attributes'].str.startswith('ID=' + gene.id),
                                              'end']) - 3*shift_num
            else:
                self.gff_corrected.loc[self.gff_corrected['attributes'].str.startswith('ID=' + gene.id), 'start'] \
                 = int(self.gff_corrected.loc[self.gff_corrected['attributes'].str.startswith('ID=' + gene.id),
                                              'start']) + 3*shift_num

    def correct_gff(self):
        print('Auto correct start')
        for gene in self.gff.features_of_type('gene', order_by='start'):
            if gene.attributes['gene_biotype'][0] == 'protein_coding':
                seq_combined = ""
                for cds in self.gff.children(gene,
                                             featuretype='CDS',
                                             order_by='start',
                                             reverse=False if gene.strand == '+' else True):
                    seq = get_seq(self.seq, cds.start, cds.end)
                    if cds.strand == '-':
                        seq_combined += seq.reverse_complement()
                    else:
                        seq_combined += seq
                try:
                    seq_combined.translate(cds=True, table=11)
                except:
                    self._correct_record(gene)
            elif gene.attributes['gene_biotype'][0] == 'tRNA':
                ge_st, ge_ed = gene.start, gene.end
                idx_list = self.pga_dialect.loc[self.pga_dialect['locus'] == gene.attributes['Name'][0], 'index'].to_list()
                if idx_list:
                    pga_st_list = self.gff_pga.loc[idx_list, 'start'].to_list()
                    pga_ed_list = self.gff_pga.loc[idx_list, 'end'].to_list()
                    if (ge_st not in pga_st_list) or (ge_ed not in pga_ed_list):
                        self._correct_record(gene)
        print('Auto correct done')

    def add_trna(self):
        add_list = []
        for locus in [x for x in self.pga_dialect['locus'].to_list() if x.startswith('trn')]:
            if locus not in self.geseq_dialect['locus'].to_list():
                add_list.append(locus)
        self._add_record(add_list)

    def renumber(self):
        self.gff_corrected.to_csv(self.gff_new_path, sep='\t', index=False, header=False)
        tmp_check = CheckCp(self.gff_new_path)
        self.gff_corrected = tmp_check.renumber(self.seq_id, self.species_pre)
        self.gff_corrected.to_csv(self.gff_new_path, sep='\t', index=False, header=False)

    def check(self):
        print('Auto check start')
        gff_new = gffutils.create_db(self.gff_new_path, ':memory:', merge_strategy='create_unique')
        for gene in gff_new.features_of_type('gene', order_by='start'):
            if gene.attributes['Name'] == ['rps12']:
                continue
            seq_combined = ""
            for cds in gff_new.children(gene,
                                        featuretype='CDS',
                                        order_by='start',
                                        reverse=False if gene.strand == '+' else True):
                seq = get_seq(self.seq, cds.start, cds.end)
                if cds.strand == '-':
                    seq_combined += seq.reverse_complement()
                else:
                    seq_combined += seq
            if seq_combined == '':
                continue
            elif seq_combined.__len__() <= 33:
                print('The CDS length of', gene.id, 'is less than 33 bp')
            try:
                seq_combined.translate(table=11, cds=True)
            except Exception as e:
                if e.__str__() == "First codon 'ACG' is not a start codon":
                    self._find_start_codon(gene, seq_combined)
                else:
                    print(gene.id, gene.attributes['Name'][0])
                    print(e)
                    self._add_pseudo(gene)
        self.gff_corrected.to_csv(self.gff_new_path, sep='\t', index=False, header=False)
        print('Auto check done')

    def check2(self):
        tmp_check = CheckCp(self.gff_new_path)
        tmp_check.check_region()

if __name__ == '__main__':
    import argparse

    # wrapper for sub-command line
    def combine(_args):
        info_table = pd.read_table(_args.info_table,
                                   names=['gff_path_ge', 'gff_path_pga', 'seq_path', 'gff_new_path', 'seq_id',
                                          'species_pre'])
        for ind, row in info_table.iterrows():
            a = CorrectGff(*row.to_list())
            a.create_geseq_dialect()
            a.create_pga_dialect()
            a.correct_gff()
            a.add_trna()
            a.renumber()
            a.check()
            a.check2()

    def rps12(_args):
        info_table = pd.read_table(_args.info_table,
                                   names=['raw_gff_path', 'new_gff_path', 'pga_gb_path',
                                          'seq_id', 'species_pre'])
        for ind, row in info_table.iterrows():
            print(os.path.basename(row['raw_gff_path']))
            tmp_check = CheckCp(row['raw_gff_path'])
            tmp_df = tmp_check.renumber(*row[['seq_id', 'species_pre']])
            try:
                tmp_df = add2_rps12(row['pga_gb_path'], tmp_df, row['species_pre'])
            except:
                print('please manually add rps12')
            finally:
                tmp_df.to_csv(row['new_gff_path'], sep='\t', index=False, header=False)

    # command line parser
    parser = argparse.ArgumentParser(
        description='Combine GeSeq and PGA annotation results ')

    subparsers = parser.add_subparsers(help='combine for combining GeSeq and PGA results; '
                                            'rps12 for adding rps12 and renumber')
    # for step6
    parser_a = subparsers.add_parser('combine', help='combine for combining GeSeq and PGA results')
    parser_a.add_argument('-i', '--info_table', required=True,
                          help='<file_path>  information table which has four columns: Geseq gff path, '
                               'result path, seqid, locus prefix')
    parser_a.set_defaults(func=combine)
    # for step8
    parser_b = subparsers.add_parser('rps12', help='adding rps12 and renumber')
    parser_b.add_argument('-i', '--info_table', required=True,
                          help='<file_path>  information table which has five columns: Raw gff path, '
                               'result path, PGA genbank file, seqid, locus prefix')
    parser_b.set_defaults(func=rps12)
    args = parser.parse_args()
    args.func(args)

max_stars_repo_name: Hua-CM/ChloroplastAnnotation

max_issues_repo_name: Hua-CM/ChloroplastAnnotation

max_forks_repo_name: Hua-CM/ChloroplastAnnotation


text:
"""
===============================
Find global rejection threshold
===============================

This example demonstrates how to use ``autoreject`` to
find global rejection thresholds.
"""

# Author: Mainak Jas <mainak.jas@telecom-paristech.fr>
# License: BSD (3-clause)

import numpy as np
import matplotlib
from sklearn.learning_curve import validation_curve
from sklearn.cross_validation import KFold

import mne
import matplotlib.pyplot as plt
from autoreject import GlobalAutoReject

from mne.datasets import sample
from mne import io

print(__doc__)

data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_filt-0-40_raw.fif'
raw = io.Raw(raw_fname, preload=True)

event_fname = data_path + ('/MEG/sample/sample_audvis_filt-0-40_raw-'
                           'eve.fif')
event_id = {'Visual/Left': 3}
tmin, tmax = -0.2, 0.5
events = mne.read_events(event_fname)

include = []
picks = mne.pick_types(raw.info, meg=False, eeg=True, stim=False,
                       eog=False, include=include, exclude='bads')
epochs = mne.Epochs(raw, events, event_id, tmin, tmax,
                    picks=picks, baseline=(None, 0),
                    reject=None, verbose=False, detrend=True)

param_range = np.linspace(400e-7, 200e-6, 30)
X = epochs.get_data()
n_epochs, n_channels, n_times = X.shape

human_thresh = 80e-6
unit = r'$\mu$V'
scaling = 1e6
cv = KFold(n_epochs, n_folds=5, random_state=42)

# XXX : you should have your own learning_curve to be able to pass Epochs
# to fit and avoid passing these n_channels, n_times to the init
_, test_scores = validation_curve(
    GlobalAutoReject(n_channels, n_times), X.reshape(n_epochs, -1), y=None,
    param_name="thresh", param_range=param_range, cv=cv, n_jobs=1,
    verbose=1)

test_scores = -test_scores.mean(axis=1)
best_thresh = param_range[np.argmin(test_scores)]

matplotlib.style.use('ggplot')
fontsize = 17
params = {'axes.labelsize': fontsize + 2,
          'text.fontsize': fontsize,
          'legend.fontsize': fontsize,
          'xtick.labelsize': fontsize,
          'ytick.labelsize': fontsize}
plt.rcParams.update(params)

plt.figure(figsize=(8, 5))
plt.tick_params(axis='x', which='both', bottom='off', top='off')
plt.tick_params(axis='y', which='both', left='off', right='off')

colors = plt.rcParams['axes.color_cycle']

plt.plot(scaling * param_range, scaling * test_scores,
         'o-', markerfacecolor='w',
         color=colors[0], markeredgewidth=2, linewidth=2,
         markeredgecolor=colors[0], markersize=8, label='CV scores')
plt.ylabel('RMSE (%s)' % unit)
plt.xlabel('Threshold (%s)' % unit)
plt.xlim((scaling * param_range[0] - 10, scaling * param_range[-1] + 10))
plt.axvline(scaling * best_thresh, label='auto global', color=colors[2],
            linewidth=2, linestyle='--')
plt.axvline(scaling * human_thresh, label='manual', color=colors[1],
            linewidth=2, linestyle=':')

plt.legend(loc='upper right')

ax = plt.gca()
plt.tight_layout()
plt.show()

ext: py

max_stars_repo_name: dengemann/autoreject

max_issues_repo_name: dengemann/autoreject

max_forks_repo_name: dengemann/autoreject

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Sat May 19 15:53:21 2018

Author: Josef Perktold
License: BSD-3
"""

from collections import defaultdict
import numpy as np

from statsmodels.base._penalties import SCADSmoothed

class ScreeningResults(object):
    """Results for Variable Screening

    Note: Indices except for exog_idx and in the iterated case also
    idx_nonzero_batches are based on the combined [exog_keep, exog] array.

    **Attributes**

    results_final : instance
        Results instance returned by the final fit of the penalized model, i.e.
        after trimming exog with params below trimming threshold.
    results_pen : results instance
        Results instance of the penalized model before trimming. This includes
        variables from the last forward selection
    idx_nonzero
        index of exog columns in the final selection including exog_keep
    idx_exog
        index of exog columns in the final selection for exog candidates, i.e.
        without exog_keep
    idx_excl
        idx of excluded exog based on combined [exog_keep, exog] array. This is
        the complement of idx_nonzero
    converged : boolean
        True if the iteration has converged and stopped before maxiter has been
        reached. False if maxiter has been reached.
    iterations : int
        number of iterations in the screening process. Each iteration consists
        of a forward selection step and a trimming step.
    history : dict of lists
        results collected for each iteration during the screening process
        'idx_nonzero' 'params_keep'].append(start_params)
            history['idx_added'].append(idx)

    The ScreeningResults returned by `screen_exog_iterator` has additional
    attributes:

    idx_nonzero_batches : ndarray 2-D
        Two-dimensional array with batch index in the first column and variable
        index withing batch in the second column. They can be used jointly as
        index for the data in the exog_iterator.
    exog_final_names : list of strings
        'var<bidx>_<idx>' where `bidx` is the batch index and `idx` is the
        index of the selected column withing batch `bidx`.
    history_batches : dict of lists
        This provides information about the selected variables within each
        batch during the first round screening
        'idx_nonzero' is based ond the array that includes exog_keep, while
        'idx_exog' is the index based on the exog of the batch.

    """
    def __init__(self, screener, **kwds):
        self.screener = screener
        self.__dict__.update(**kwds)

class VariableScreening(object):
    """Ultra-high, conditional sure independence screening

    This is an adjusted version of Fan's sure independence screening.

    Parameters
    ----------
    model : instance of penalizing model
        examples: GLMPenalized, PoissonPenalized and LogitPenalized.
        The attributes of the model instance `pen_weight` and `penal` will be
        ignored.
    pen_weight : None or float
        penalization weight use in SCAD penalized MLE
    k_add : int
        number of exog to add during expansion or forward selection
        see Notes section for tie handling
    k_max_add : int
        maximum number of variables to include during variable addition, i.e.
        forward selection. default is 30
    threshold_trim : float
        threshold for trimming parameters to zero, default is 1e-4
    k_max_included : int
        maximum total number of variables to include in model.
    ranking_attr : str
        This determines the result attribute or model method that is used for
        the ranking of exog to include. The availability of attributes depends
        on the model.
        Default is 'resid_pearson', 'model.score_factor' can be used in GLM.
    ranking_project : bool
        If ranking_project is True, then the exog candidates for inclusion are
        first projected on the already included exog before the computation
        of the ranking measure. This brings the ranking measure closer to
        the statistic of a score test for variable addition.

    Notes
    -----
    Status: experimental, tested only on a limited set of models and
    with a limited set of model options.

    Tie handling: If there are ties at the decision threshold, then all those
    tied exog columns are treated in the same way. During forward selection
    all exog columns with the same boundary value are included. During
    elimination, the tied columns are not dropped. Consequently, if ties are
    present, then the number of included exog can be larger than specified
    by k_add, k_max_add and k_max_included.

    The screening algorithm works similar to step wise regression. Each
    iteration of the screening algorithm includes a forward selection step
    where variables are added to the model, and a backwards selection step
    where variables are removed. In contrast to step wise regression, we add
    a fixed number of variables at each forward selection step. The
    backwards selection step is based on SCAD penalized estimation and
    trimming of variables with estimated coefficients below a threshold.
    The tuning parameters can be used to adjust the number of variables to add
    and to include depending on the size of the dataset.

    There is currently no automatic tuning parameter selection. Candidate
    explanatory variables should be standardized or should be on a similar
    scale because penalization and trimming are based on the absolute values
    of the parameters.

    TODOs and current limitations:

    freq_weights are not supported in this. Candidate ranking uses
    moment condition with resid_pearson or others without freq_weights.
    pearson_resid: GLM resid_pearson does not include freq_weights.

    variable names: do we keep track of those? currently made-up names

    currently only supports numpy arrays, no exog type check or conversion

    currently only single columns are selected, no terms (multi column exog)

    """

    def __init__(self, model, pen_weight=None, use_weights=True, k_add=30,
                 k_max_add=30, threshold_trim=1e-4, k_max_included=20,
                 ranking_attr='resid_pearson', ranking_project=True):

        self.model = model
        self.model_class = model.__class__
        self.init_kwds = model._get_init_kwds()
        # pen_weight and penal are explicitly included
        # TODO: check what we want to do here
        self.init_kwds.pop('pen_weight', None)
        self.init_kwds.pop('penal', None)

        self.endog = model.endog
        self.exog_keep = model.exog
        self.k_keep = model.exog.shape[1]
        self.nobs = len(self.endog)
        self.penal = self._get_penal()

        if pen_weight is not None:
            self.pen_weight = pen_weight
        else:
            self.pen_weight = self.nobs * 10

        # option for screening algorithm
        self.use_weights = use_weights
        self.k_add = k_add
        self.k_max_add = k_max_add
        self.threshold_trim = threshold_trim
        self.k_max_included = k_max_included
        self.ranking_attr = ranking_attr
        self.ranking_project = ranking_project

    def _get_penal(self, weights=None):
        """create new Penalty instance
        """
        return SCADSmoothed(0.1, c0=0.0001, weights=weights)

    def ranking_measure(self, res_pen, exog, keep=None):
        """compute measure for ranking exog candidates for inclusion

        """
        endog = self.endog

        if self.ranking_project:
            assert res_pen.model.exog.shape[1] == len(keep)
            ex_incl = res_pen.model.exog[:, keep]
            exog = exog - ex_incl.dot(np.linalg.pinv(ex_incl).dot(exog))

        if self.ranking_attr == 'predicted_poisson':
            # I keep this for more experiments

            # TODO: does it really help to change/trim params
            # we are not reestimating with trimmed model
            p = res_pen.params.copy()
            if keep is not None:
                p[~keep] = 0
            predicted = res_pen.model.predict(p)
            # this is currently hardcoded for Poisson
            resid_factor = (endog - predicted) / np.sqrt(predicted)
        elif self.ranking_attr[:6] == 'model.':
            # use model method, this is intended for score_factor
            attr = self.ranking_attr.split('.')[1]
            resid_factor = getattr(res_pen.model, attr)(res_pen.params)
            if resid_factor.ndim == 2:
                # for score_factor when extra params are in model
                resid_factor = resid_factor[:, 0]
            mom_cond = np.abs(resid_factor.dot(exog))**2
        else:
            # use results attribute
            resid_factor = getattr(res_pen, self.ranking_attr)
            mom_cond = np.abs(resid_factor.dot(exog))**2
        return mom_cond

    def screen_exog(self, exog, endog=None, maxiter=100, method='bfgs',
                    disp=False, fit_kwds=None):
        """screen and select variables (columns) in exog

        Parameters
        ----------
        exog : ndarray
            candidate explanatory variables that are screened for inclusion in
            the model
        endog : ndarray (optional)
            use a new endog in the screening model.
            This is not tested yet, and might not work correctly
        maxiter : int
            number of screening iterations
        method : str
            optimization method to use in fit, needs to be only of the gradient
            optimizers
        disp : bool
            display option for fit during optimization

        Returns
        -------
        res_screen : instance of ScreeningResults
            The attribute `results_final` contains is the results instance
            with the final model selection.
            `idx_nonzero` contains the index of the selected exog in the full
            exog, combined exog that are always kept plust exog_candidates.
            see ScreeningResults for a full description

        """
        model_class = self.model_class
        if endog is None:
            # allow a different endog than used in model
            endog = self.endog
        x0 = self.exog_keep
        k_keep = self.k_keep
        x1 = exog
        k_current = x0.shape[1]
        # TODO: remove the need for x, use x1 separately from x0
        # needs change to idx to be based on x1 (candidate variables)
        x = np.column_stack((x0, x1))
        nobs, k_vars = x.shape
        fkwds = fit_kwds if fit_kwds is not None else {}
        fit_kwds = {'maxiter': 200, 'disp': False}
        fit_kwds.update(fkwds)

        history = defaultdict(list)
        idx_nonzero = np.arange(k_keep, dtype=np.int)
        keep = np.ones(k_keep, np.bool_)
        idx_excl = np.arange(k_keep, k_vars)
        mod_pen = model_class(endog, x0, **self.init_kwds)
        # don't penalize initial estimate
        mod_pen.pen_weight = 0
        res_pen = mod_pen.fit(**fit_kwds)
        start_params = res_pen.params
        converged = False
        idx_old = []
        for it in range(maxiter):
            # candidates for inclusion in next iteration
            x1 = x[:, idx_excl]
            mom_cond = self.ranking_measure(res_pen, x1, keep=keep)
            assert len(mom_cond) == len(idx_excl)
            mcs = np.sort(mom_cond)[::-1]
            idx_thr = min((self.k_max_add, k_current + self.k_add, len(mcs)))
            threshold = mcs[idx_thr]
            # indices of exog in current expansion model
            idx = np.concatenate((idx_nonzero, idx_excl[mom_cond > threshold]))
            start_params2 = np.zeros(len(idx))
            start_params2[:len(start_params)] = start_params

            if self.use_weights:
                weights = np.ones(len(idx))
                weights[:k_keep] = 0
                # modify Penalty instance attached to self
                # damgerous if res_pen is reused
                self.penal.weights = weights
            mod_pen = model_class(endog, x[:, idx], penal=self.penal,
                                  pen_weight=self.pen_weight,
                                  **self.init_kwds)

            res_pen = mod_pen.fit(method=method,
                                  start_params=start_params2,
                                  warn_convergence=False, skip_hessian=True,
                                  **fit_kwds)

            keep = np.abs(res_pen.params) > self.threshold_trim
            # use largest params to keep
            if keep.sum() > self.k_max_included:
                # TODO we can use now np.partition with partial sort
                thresh_params = np.sort(np.abs(res_pen.params))[
                                                        -self.k_max_included]
                keep2 = np.abs(res_pen.params) > thresh_params
                keep = np.logical_and(keep, keep2)

            # Note: idx and keep are for current expansion model
            # idx_nonzero has indices of selected variables in full exog
            keep[:k_keep] = True  # always keep exog_keep
            idx_nonzero = idx[keep]

            if disp:
                print(keep)
                print(idx_nonzero)
            # x0 is exog of currently selected model, not used in iteration
            # x0 = x[:, idx_nonzero]
            k_current = len(idx_nonzero)
            start_params = res_pen.params[keep]

            # use mask to get excluded indices
            mask_excl = np.ones(k_vars, dtype=bool)
            mask_excl[idx_nonzero] = False
            idx_excl = np.nonzero(mask_excl)[0]
            history['idx_nonzero'].append(idx_nonzero)
            history['keep'].append(keep)
            history['params_keep'].append(start_params)
            history['idx_added'].append(idx)

            if (len(idx_nonzero) == len(idx_old) and
                    (idx_nonzero == idx_old).all()):
                converged = True
                break
            idx_old = idx_nonzero

        # final esimate
        # check that we still have exog_keep
        assert np.all(idx_nonzero[:k_keep] == np.arange(k_keep))
        if self.use_weights:
            weights = np.ones(len(idx_nonzero))
            weights[:k_keep] = 0
            # create new Penalty instance to avoide sharing attached penal
            penal = self._get_penal(weights=weights)
        else:
            penal = self.penal
        mod_final = model_class(endog, x[:, idx_nonzero],
                                penal=penal,
                                pen_weight=self.pen_weight,
                                **self.init_kwds)

        res_final = mod_final.fit(method=method,
                                  start_params=start_params,
                                  warn_convergence=False,
                                  **fit_kwds)
        # set exog_names for final model
        xnames = ['var%4d' % ii for ii in idx_nonzero]
        res_final.model.exog_names[k_keep:] = xnames[k_keep:]

        res = ScreeningResults(self,
                               results_pen = res_pen,
                               results_final = res_final,
                               idx_nonzero = idx_nonzero,
                               idx_exog = idx_nonzero[k_keep:] - k_keep,
                               idx_excl = idx_excl,
                               history = history,
                               converged = converged,
                               iterations = it + 1  # it is 0-based
                               )
        return res

    def screen_exog_iterator(self, exog_iterator):
        """
        batched version of screen exog

        This screens variables in a two step process:

        In the first step screen_exog is used on each element of the
        exog_iterator, and the batch winners are collected.

        In the second step all batch winners are combined into a new array
        of exog candidates and `screen_exog` is used to select a final
        model.

        Parameters
        ----------
        exog_iterator : iterator over ndarrays

        Returns
        -------
        res_screen_final : instance of ScreeningResults
            This is the instance returned by the second round call to
            `screen_exog`. Additional attributes are added to provide
            more information about the batched selection process.
            The index of final nonzero variables is
            `idx_nonzero_batches` which is a 2-dimensional array with batch
            index in the first column and variable index within batch in the
            second column. They can be used jointly as index for the data
            in the exog_iterator.
            see ScreeningResults for a full description

        """
        k_keep = self.k_keep
        # res_batches = []
        res_idx = []
        exog_winner = []
        exog_idx = []
        for ex in exog_iterator:
            res_screen = self.screen_exog(ex, maxiter=20)
            # avoid storing res_screen, only for debugging
            # res_batches.append(res_screen)
            res_idx.append(res_screen.idx_nonzero)
            exog_winner.append(ex[:, res_screen.idx_nonzero[k_keep:] - k_keep])
            exog_idx.append(res_screen.idx_nonzero[k_keep:] - k_keep)

        exog_winner = np.column_stack(exog_winner)
        res_screen_final = self.screen_exog(exog_winner, maxiter=20)

        exog_winner_names = ['var%d_%d' % (bidx, idx)
                             for bidx, batch in enumerate(exog_idx)
                             for idx in batch]

        idx_full = [(bidx, idx)
                    for bidx, batch in enumerate(exog_idx)
                    for idx in batch]
        ex_final_idx = res_screen_final.idx_nonzero[k_keep:] - k_keep
        final_names = np.array(exog_winner_names)[ex_final_idx]
        res_screen_final.idx_nonzero_batches = np.array(idx_full)[ex_final_idx]
        res_screen_final.exog_final_names = final_names
        history = {'idx_nonzero': res_idx,
                   'idx_exog': exog_idx}
        res_screen_final.history_batches = history
        return res_screen_final

max_stars_repo_name: nikhase/statsmodels

max_issues_repo_name: nikhase/statsmodels

max_forks_repo_name: nikhase/statsmodels

reason: import numpy,from statsmodels


text:
import os, rasterio
import xarray as xr
from shapely.geometry import Point
import numpy as np
import pandas as pd
import geopandas as gpd
import glob

'''
Extract time-series data for individual points from downscaled Alaska ERA Interim
data (Bieniek et al., 2015). Input file is a netCDF files.
Code partially (mostly?) courtesy of Michael Lindgren at UAF.
'''

def extract_loc_profile(path_to_file, lat, lon, variable, loc_name):

    '''
    path_to_file = path to file (string)
    lat = latitude of point of interest (float)
    lon = longitude of point of interest (float)
    variable = name of dataset variable, i.e. pcpt, t2 etc. (string)
    loc_name = Location name, i.e. Fairbanks, Columbia Glacier, (string)
    '''

    # read in a NetCDF dataset using xarray
    ds = xr.open_dataset( path_to_file )

    # point location in EPSG:4326 -- WGS84 latlon -- Flat Creek Glacier
    lon,lat = ( lon, lat )

    # some constant metadata about the WRF grid
    res = 20000 # 20km resolution both directions
    x0,y0 = np.array( ds.xc.min()-(res/2.)), np.array(ds.yc.max()+(res/2.) ) # origin point upper-left corner from centroid
    wrf_crs = '+units=m +proj=stere +lat_ts=64.0 +lon_0=-152.0 +lat_0=90.0 +x_0=0 +y_0=0 +a=6370000 +b=6370000'
    a = rasterio.transform.from_origin( x0, y0, res, res ) # build affine transform using rasterio mechanics

    # make the point location in to a geopandas GeoDataFrame -- there are ways to do this with pyproj, but I like this approach
    shp = gpd.GeoDataFrame({'id':[1], 'name':[loc_name], 'geometry':[Point(lon,lat)]}, crs={'init':'epsg:4326'}, geometry='geometry' )

    # reproject that point shapefile to the wrf crs
    shp_wrf = shp.to_crs( wrf_crs )

    # now lets pull out the x,y from that point object in the reprojected shapefile
    xy, = shp_wrf.geometry.tolist()
    x = xy.x
    y = xy.y

    # now we can use the affine transform with these x/y values to find where in space we land
    col, row = ~a * (x, y)
    col, row = [ int(i) for i in [col, row] ] # truncate to integer

    # now using that info we can pull a profile from the NetCDF using NumPy ndarray indexing.
    profile = ds[variable][:,row,col]

    # then you can turn it into a pandas object (a series in this case) with this
    #series = profile.to_pandas().head()

    # or to a pandas dataframe (with all of the coordinate information) like this
    df = profile.to_dataframe()

    return(df,x,y,col,row)

files = glob.glob('t2*')
#print(files)

time_series = pd.DataFrame([])
#print(time_series)

#FC coordinates -141.557730, 61.640682
#Chisana coordinates -142.042191,62.068633

for infile in files:
	print(infile)
	df,x,y,col,row = extract_loc_profile(infile, 62.068633, -142.042191, 't2', 'Chisana')
	#print(df,x,y,col,row)
	time_series = time_series.append(df)
	#print(time_series)

time_series = time_series.sort_index(axis = 0)
time_series.to_csv('./ERA_t2_hourly_UTC_Chisana.csv')

max_stars_repo_name: mjacqu/FlatCreekProject

max_issues_repo_name: mjacqu/FlatCreekProject

max_forks_repo_name: mjacqu/FlatCreekProject


text:
from picamera.array import PiRGBArray
from picamera import PiCamera
import time
import cv2
import os
import numpy as np
from  PIL import Image
import pickle
import pygame
from pygame.locals import *
import RPi.GPIO as GPIO
import subprocess
import sys
import pygame.mixer
import smtplib
from weather import Weather, Unit
from importlib import import_module
import os
from flask import Flask, render_template, Response, request
import io
import picamera
from base_camera import BaseCamera
import itchat

@itchat.msg_register(itchat.content.TEXT)
def get_msg(msg):
    global status_flag
    if msg.text == 'no':
        status_flag = 0
        itchat.logout()
    if msg.text == 'yes':
        status_flag = 3
        itchat.logout()
    print msg.text

class Camera(BaseCamera):
    global camera
    global camera_flag
    global x
    global y
    @staticmethod
    def frames():
        #with picamera.PiCamera() as camera:
            # let camera warm up

        while camera_flag:
            time.sleep(2)

            stream = io.BytesIO()
            for _ in camera.capture_continuous(stream, 'jpeg',
                                                 use_video_port=True):
                if camera_flag < 1:
                    break
                # return current frame
                stream.seek(0)
                yield stream.read()

                # reset stream for next frame
                stream.seek(0)
                stream.truncate()

app = Flask(__name__)

@app.route("/", methods=['GET', 'POST'])
def index():
    global camera_flag
    global status_flag
    global start_web
    #if (time.time() - start_web) > 10:
    #    func = request.environ.get('werkzeug.server.shutdown')
    #    func()
        #shutdown_server()
    #    return

    """Video streaming home page."""
    print(request.method)
    if request.method == 'POST':
        if request.form.get('Open') == 'Open':
                # pass
            print("Open")
            #camera_flag = 0
            status_flag = 3
            #open_flag
            func = request.environ.get('werkzeug.server.shutdown')
            func()
            #shutdown_server()
            return render_template("success.html")
        elif  request.form.get('Reject') == 'Reject':
                # pass # do something else
            status_flag = 0
            print("Reject")
            func = request.environ.get('werkzeug.server.shutdown')
            func()
            return render_template("fail.html")
        else:
                # pass # unknown
            return render_template("index.html")
    elif request.method == 'GET':
            # return render_template("index.html")
            print("No Post Back Call")
    return render_template("index.html")

def gen(camera):
    """Video streaming generator function."""
    while True:
        frame = camera.get_frame()
        yield (b'--frame\r\n'
               b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')

@app.route('/video_feed')
def video_feed():
    """Video streaming route. Put this in the src attribute of an img tag."""
    return Response(gen(Camera()),
                    mimetype='multipart/x-mixed-replace; boundary=frame')

@app.route('/contact',methods=['GET', 'POST'])
def contact():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Do Something':
            print("a")
            pass # do something
        elif request.form['submit_button'] == 'Do Something Else':
            print("b")
            pass # do something else
        else:
            pass # unknown
    elif request.method == 'GET':
        return render_template('contact.html', form=form)
##################################################################################################################
##########################################################
##################################################################################################################
def sendm():
    smtpUser = 'hs978xh344@gmail.com'
    smtpPass = 'ece5725!'

    toAdd='hs978@cornell.edu'#'xh344@cornell.edu'
    fromAdd = smtpUser

    subject = 'knock,knock http://172.20.10.4:5000/'#http://128.253.17.23:5000/'
    header = 'To: ' + toAdd + '\n' + 'From: ' + fromAdd + '\n' + 'Subject: ' + subject
    link='128,253,17,23:5000'
    body = 'check who is coming? '

    print header + '\n' + body

    s = smtplib.SMTP('smtp.gmail.com',587)
    s.ehlo()
    s.starttls()
    s.ehlo()

    s.login(smtpUser,smtpPass)
    s.sendmail(fromAdd, toAdd, header + '\n' + body)
    s.quit()
#white = 255,255,255
#black = 0,0,0

def register(name, str):
    file=open('/home/pi/final_project/password/'+name+'.txt',"w")
    file.write(str)

def changepw(str):
    file=open('/home/pi/final_project/password/hs.txt',"w")
    file.write(str)

def checkpw(id_, str):
    #str is the user input in GUI
    #if(id_==0):
        with open("/home/pi/final_project/password/hs.txt") as file:
            data1=file.read()

                #unlock the door call servo function

    #if(id_==1):
        with open("/home/pi/final_project/password/hxh.txt") as file:
            data2=file.read()

        return data1==str or data2==str

    #unlock the door call servo function

def train():
    recognizer = cv2.createLBPHFaceRecognizer()
    cascadePath= "/home/pi/final_project/opencv-master/data/haarcascades/haarcascade_frontalface_alt2.xml"
    face_cascade =cv2.CascadeClassifier(cascadePath)
# find the image directory
    BASE_DIR =os.path.dirname("/home/pi/final_project")
    image_dir =os.path.join(BASE_DIR, "final_project/images")
#print(image_dir)

#initialize the the trauning variable
    curr_id=0
    label_ids ={}
    y_labels=[]
    x_train=[]

    for root,dirs,files in os.walk(image_dir):

        for file in files :

           if file.endswith("jpg"):
                path = os.path.join(root,file)
                label =os.path.basename(root)
            #print(label)

                if not label in label_ids:
                    label_ids[label] = curr_id
                    curr_id=curr_id+1
                id_= label_ids[label]
            #print(label_ids)
                pil_image=Image.open(path).convert("L")# grayscale
                image_array = np.array(pil_image,"uint8")
            #print(image_array)
                faces=face_cascade.detectMultiScale(image_array, scaleFactor=1.5, minNeighbors=5,minSize=(5,5), flags= cv2.cv.CV_HAAR_SCALE_IMAGE)

                for(x,y,w,h) in faces:
                    roi= image_array[y:y+h,x:x+w]
                    x_train.append(roi)
                    y_labels.append(id_)

#print(x_train) just for check
    print(y_labels)
# store the label
    with open("labels.pickle",'wb') as f:
        pickle.dump(label_ids,f)

#start to train
    recognizer.train(x_train,np.array(y_labels))
    recognizer.save("trainer.yml")

def gather_info():
    i=0
    im_num = 80

    rawCapture = PiRGBArray(camera,size=(480,320))
    face_cascade=cv2.CascadeClassifier('/home/pi/final_project/opencv-master/data/haarcascades/haarcascade_frontalface_alt2.xml')
    #time_start_l = time.time()
    for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port =True):
	#time_interval_l = time_now_l - time_start_l
	#if time_interval_l > 60:
        #    status_flag = 2

        #    return 0

	frame= frame.array
	gray =cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
	faces=face_cascade.detectMultiScale(gray, scaleFactor=1.5, minNeighbors=5,minSize=(5,5), flags= cv2.cv.CV_HAAR_SCALE_IMAGE)
	for(x,y,w,h) in faces:
		print(x,y,w,h)
		rog= gray[y:y+h,x:x+w]# region of interets in gray scale

	    # recognizer

		color1= (255,0,0)
		color2= (0,255,0)
		color3= (0,0,255)
		im=frame

		if(i<=im_num ):
                    cv2.imwrite('/home/pi/final_project/images/user2/'+str(i)+'.jpg', frame)#[y:y+h,x:x+h])
                    i=i+1
                else:
                    return 1
                cv2.rectangle(im,(x,y),(x+w,y+w),color1,2)
                    #cv2.imwrite('/home/pi/Desktop/test.jpg', frame[y:y+h,x:x+h])
                    #i = i+1
	#cv2.imshow('frame', frame)
        screen.fill(BLACK)################################
        sf=cv2.resize(frame,(280,240))
        sf= cv2.cvtColor(sf,cv2.COLOR_BGR2RGB)
        sf=np.rot90(sf)
        sf=pygame.surfarray.make_surface(sf)
        screen.blit(sf,(0,0))
        height = (240 * i) / im_num
        #print(height)
        pygame.draw.rect(screen,WHITE,[280,240 - height,40,height],0)
        pygame.display.flip()
	rawCapture.truncate(0)
    return 0
######################################################################
#   Function name: face_rec
#
#   Description: used for face detection in real time
#                will break when a face is successfully detected
#
#   Made by: xh344
#
#   Date: 2018.11.27
######################################################################

def face_rec():
    # enable the picamera
    global status_flag
    global screen#################################
    global camera_flag
    global camera
    #if camera_flag == 0:

    rawCapture = PiRGBArray(camera,size=(480,320))

    record=0
    sum=0
    array=[]
    #input the face_cascade
    face_cascade=cv2.CascadeClassifier('/home/pi/final_project/opencv-master/data/haarcascades/haarcascade_frontalface_alt2.xml')
    #initialize the facerecognizer
    recognizer = cv2.createLBPHFaceRecognizer()
    #load the traning data
    recognizer.load("trainer.yml")
    i = 0
    time_start_l = time.time()
    # capture frame by frame and detect and recognize the face frame by frame
    for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port =True):

	frame= frame.array
	gray =cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
	faces=face_cascade.detectMultiScale(gray, scaleFactor=1.5, minNeighbors=5,minSize=(5,5), flags= cv2.cv.CV_HAAR_SCALE_IMAGE)
	time_now_l = time.time()
	time_interval_l = time_now_l - time_start_l
	if time_interval_l > 15:
            status_flag = 2
#            camera.close()
            return -1
	for(x,y,w,h) in faces:
		print(x,y,w,h)
	    # region of interets in gray scale
		rog= gray[y:y+h,x:x+w]

	    # recognizer predict he id and the confidence number which associate with
                id_, conf = recognizer.predict(rog)

            # after mutiple adjusting the confidence number, we use this region
                if conf>= 35 and conf<=88:
                    print(id_)
                    array.append(id_)
                    if len(array)<=5:
                        sum=sum+id_

                    else:
                        if sum>5:
                            status_flag = 3
                            #camera.close()
                            return 1
                        else:
                            status_flag = 3
                            #camera.close()
                            return 0

                    #status_flag = 3
                    #return -1
                    ###start to call checkpw function conbine with gui
	    # for different colors of retangular
		color1= (255,0,0)
		color2= (0,255,0)
		color3= (0,0,255)
		cv2.rectangle(frame,(x,y),(x+w,y+w),color1,2)
		#if(record == 0 ):
                    #cv2.imwrite('/home/pi/Desktop/datah/test'+str(i)+'.jpg', frame)#[y:y+h,x:x+h])
                    #cv2.imwrite('/home/pi/Desktop/test.jpg', frame[y:y+h,x:x+h])
                    #i = i+1
	#cv2.imshow('frame', frame)

	screen.fill(BLACK)################################
        sf=cv2.resize(frame,(320,240))
        sf= cv2.cvtColor(sf,cv2.COLOR_BGR2RGB)
        sf=np.rot90(sf)
        sf=pygame.surfarray.make_surface(sf)
        screen.blit(sf,(0,0))
        pygame.display.flip()

	rawCapture.truncate(0)

	# physical key for quiting
	if cv2.waitKey(20) & 0xFF ==ord('q'):
            break
    status_flag = 2
    #camera.close()
    return -1
######################################################################
#fun face_rec end here
######################################################################

def opend():
    global p_left
    global p_right

    vdd= 1.53
    dc = 100 * vdd / (vdd + 20)
    f = 1000 / (vdd + 20)

    vdd2= 1.47
    dc2 = 100 * vdd2 / (vdd2 + 20)
    f2 = 1000 / (vdd2 + 20)

    vdd3= 0
    dc3 = 100 * vdd3 / (vdd3 + 20)
    f3 = 1000 / (vdd3 + 20)

    p_left.ChangeFrequency(f)
    p_left.ChangeDutyCycle(dc)

    p_right.ChangeFrequency(f2)
    p_right.ChangeDutyCycle(dc2)

    #p_left.start(dc)
    #p_right.start(dc2)
    time_start=time.time()

    while 1:
        time_now=time.time()
        time_interval=time_now-time_start
        if time_interval > 1.2:
            break

    p_left.ChangeFrequency(f3)
    p_left.ChangeDutyCycle(dc3)

    p_right.ChangeFrequency(f3)
    p_right.ChangeDutyCycle(dc3)

def closed():
    global p_left
    global p_right

    vdd= 1.53
    dc = 100 * vdd / (vdd + 20)
    f = 1000 / (vdd + 20)

    vdd2= 1.47
    dc2 = 100 * vdd2 / (vdd2 + 20)
    f2 = 1000 / (vdd2 + 20)

    vdd3= 0
    dc3 = 100 * vdd3 / (vdd3 + 20)
    f3 = 1000 / (vdd3 + 20)

    p_left.ChangeFrequency(f2)
    p_left.ChangeDutyCycle(dc2)

    p_right.ChangeFrequency(f)
    p_right.ChangeDutyCycle(dc)

    #p_left.start(dc2)
    #p_right.start(dc)
    time_start=time.time()

    while 1:
        time_now=time.time()
        time_interval=time_now-time_start
        if time_interval > 1:
            break

    #p_left.stop()
    #p_right.stop()
    p_left.ChangeFrequency(f3)
    p_left.ChangeDutyCycle(dc3)

    p_right.ChangeFrequency(f3)
    p_right.ChangeDutyCycle(dc3)

os.putenv('SDL_VIDEODRIVER', 'fbcon') # Display on piTFT
os.putenv('SDL_FBDEV', '/dev/fb1')
os.putenv('SDL_MOUSEDRV', 'TSLIB') # Track mouse clicks on piTFT
os.putenv('SDL_MOUSEDEV', '/dev/input/touchscreen')
GPIO.setmode(GPIO.BCM)
GPIO.setup(26, GPIO.IN)
GPIO.setup(5, GPIO.OUT)
GPIO.setup(13, GPIO.OUT)
vdd= 0
dc = 100 * vdd / (vdd + 20)
f = 1000 / (vdd + 20)

vdd2= 0
dc2 = 100 * vdd2 / (vdd2 + 20)
f2 = 1000 / (vdd2 + 20)

#vdd3 = 0
#dc3 = 100 * vdd3 / (vdd3 + 20)
#f3 = 1000 / (vdd3 + 20)

p_left = GPIO.PWM(5,f)
p_left.start(dc2)
p_right = GPIO.PWM(13,f2)
p_right.start(dc2)

camera= PiCamera()
#camera = pygame.camera.Camera()
camera.resolution=(480,320)
camera.framerate=64
#camera_flag = 1
#register("hxh",str(1111))
#register("hs",str(1111))
s1 = "espeak '"
s = "hello"
s3 = "' --stdout | aplay -D plughw:1,0"
str_out = s1 + s + s3
os.system("espeak 'system starts' --stdout | aplay -D plughw:1,0")
pygame.init()
#pygame.camera.init()
pygame.mouse.set_visible(False)
WHITE = 255, 255, 255
BLACK = 0,0,0
size = width, height =320,240
screen = pygame.display.set_mode(size)
sleep_time=0.003

my_font_big = pygame.font.Font(None, 30)
my_font_key = pygame.font.Font(None, 20)
my_font_info = pygame.font.Font(None, 14)
my_font_num = pygame.font.Font(None, 60)

my_button_s1 = {(240,60):'Enter',(240,180):'Register',(60,180):'Quit',(150,180):'Voice',(240,120):'SetPW',(60,120):'Wechat',(150,120):'Knock'}
my_inform = {(80,180):'Press Button to enter'}
my_warning = {(160,120):'Facial Rec...'}
my_warning1 = {(160,120):'Processing...'}
my_warning2 = {(160,120):'^_^ Welcome ^_^'}
my_warning3 = {(160,120):'Waiting for web control'}
my_warning4 = {(160,100):'E-mail has been sent',(160,140):'Waiting for response throught port 5000'}
my_warning5 = {(160,120):'Please speak'}

my_number = {(30,90):'7',(30,150):'4',(30,210):'1',(90,90):'8',(90,150):'5',(90,210):'2',(150,90):'9',(150,150):'6',(150,210):'3',(210,150):'0'}
my_main_button = {(280,30):'M',(280,90):'A',(280,150):'I',(280,210):'N'}
my_code = {(30,30):'',(90,30):'',(150,30):'',(210,30):''}

my_message1 = {(160,7):'logining in to Wechat...'}
my_message2 = {(160,7):'logining in to Wechat...',(160,21):'login succeed',(160,35):'Sending message...'}
my_message3 = {(160,7):'logining in to Wechat...',(160,21):'login succeed',(160,35):'Sending message...',(160,49):'Message sent, waiting for response'}

x = 0
y = 0
flag_sys = 1
time_start=time.time()

mic = pygame.image.load("mic.png")
mic_rect = mic.get_rect()
mic_rect.center = [160,120]

snow = pygame.image.load("snow.png")
snow_rect = snow.get_rect()
snow_rect.center = [150,60]

thunder = pygame.image.load("thunder.png")
thunder_rect = thunder.get_rect()
thunder_rect.center = [150,60]

fog = pygame.image.load("fog.png")
fog_rect = fog.get_rect()
fog_rect.center = [150,60]

cloud = pygame.image.load("cloudy.png")
cloud_rect = cloud.get_rect()
cloud_rect.center = [150,60]

sun = pygame.image.load("sun.png")
sun_rect = sun.get_rect()
sun_rect.center = [150,60]

arrow = pygame.image.load("arrow.png")
arrow_rect = arrow.get_rect()
arrow_rect.center = [210,210]

deletea = pygame.image.load("delete.png")
deletea_rect = arrow.get_rect()
deletea_rect.center = [210,90]

#Degree = {(40,60):'2 CEL'}

weather_flag = 1
status_flag = 0
ID = 1
camera_flag = 0
speak_flag = 0
open_flag = 0

enter_num = 0

weather = Weather(unit = Unit.CELSIUS)
location = weather.lookup_by_location('Ithaca')
code = location.condition.code
if code < 5:#thunder
    weather_flag = 0
if code > 4 and code < 13:#snow
    weather_flag = 1
if code > 12 and code < 19:#rain
    weather_flag = 2
if code > 18 and code < 23:#foggy
    weather_flag = 3
if code > 22 and code < 31:#cloudy
    weather_flag = 4
if code > 30 and code < 37:#sunny
    weather_flag = 5
if code > 36:#thunder
    weather_flag = 0
#weather_flag = 5
temp = str(location.condition.temp) + ' CEL'
Degree = {(60,50):temp}
my_loc = {(60,70):'Ithaca, NY'}

try:
    while flag_sys:
        while status_flag == 0:
            screen.fill(BLACK)
            time_now = time.time()
            pygame.draw.rect(screen,WHITE,[210,40,60,40],0)
            pygame.draw.rect(screen,WHITE,[210,160,60,40],0)
            pygame.draw.rect(screen,WHITE,[210,100,60,40],0)
            pygame.draw.rect(screen,WHITE,[30,100,60,40],0)
            pygame.draw.rect(screen,WHITE,[120,100,60,40],0)
            pygame.draw.rect(screen,WHITE,[30,160,60,40],0)
            pygame.draw.rect(screen,WHITE,[120,160,60,40],0)

            localtime = time.asctime(time.localtime(time.time()))
            #time display
            text_surface = my_font_key.render(localtime, True, WHITE)
            rect = text_surface.get_rect(center=(50,10))
            screen.blit(text_surface, rect)
            #print(type(localtime))
            for text_pos, my_text in Degree.items():
                text_surface = my_font_big.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            for text_pos, my_text in my_button_s1.items():
                text_surface = my_font_key.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            for text_pos, my_text in my_loc.items():
                text_surface = my_font_key.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            #for text_pos, my_text in my_inform.items():
            #    text_surface = my_font_key.render(my_text, True, WHITE)
            #    rect = text_surface.get_rect(center=text_pos)
            #    screen.blit(text_surface, rect)

            if(weather_flag == 0):
                #screen.fill(BLACK)
                screen.blit(thunder, thunder_rect)
            if(weather_flag == 1):
                #screen.fill(BLACK)
                screen.blit(rain, rain_rect)
            if(weather_flag == 2):
                #screen.fill(BLACK)
                screen.blit(snow, snow_rect)
            if(weather_flag == 3):
                #screen.fill(BLACK)
                screen.blit(fog, fog_rect)
            if(weather_flag == 4):
                #screen.fill(BLACK)
                screen.blit(cloudy, cloudy_rect)
            if(weather_flag == 5):
                #screen.fill(BLACK)
                screen.blit(sun, sun_rect)

            pygame.display.flip()

            for event in pygame.event.get():
                if(event.type is MOUSEBUTTONDOWN):
                    pos = pygame.mouse.get_pos()
                elif(event.type is MOUSEBUTTONUP):
                    pos = pygame.mouse.get_pos()
                    x,y = pos

            if x > 210 and x < 270 and y > 40 and y < 80:
                status_flag = 1
                x = 0
                y = 0

            if x > 210 and x < 270 and y > 160 and y < 200:
                status_flag = 4
                x = 0
                y = 0

            if x > 210 and x < 270 and y > 100 and y < 140:
                status_flag = 6
                x = 0
                y = 0

            if x > 120 and x < 180 and y > 100 and y < 140:
                status_flag = 8
                x = 0
                y = 0

            if x > 210 and x < 270 and y > 100 and y < 140:
                status_flag = 9
                x = 0
                y = 0

            if x > 30 and x < 90 and y > 100 and y < 140:
                status_flag = 10
                x = 0
                y = 0

            if x > 120 and x < 180 and y > 160 and y < 200:
                status_flag = 11
                x = 0
                y = 0

            if x > 30 and x < 90 and y > 160 and y < 200:
                status_flag = 12
                flag_sys = 0
                x = 0
                y = 0

        while status_flag == 1:
            s = "Hi, please look at the camera!"
            str_out = s1 + s + s3
            os.system(str_out)
            screen.fill(BLACK)
            for text_pos, my_text in my_warning.items():
                text_surface = my_font_big.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            pygame.display.flip()
            ID = face_rec()
            print("out")

        speak_flag == 0
        while status_flag == 2:
            if speak_flag == 0:
                s = "Sorry, can't recognize your face, please enter password to get in"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)

            screen.fill(BLACK)
            pygame.draw.rect(screen,WHITE,[240,0,80,240],0)
            for text_pos, my_text in my_number.items():
                text_surface = my_font_num.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_main_button.items():
                text_surface = my_font_num.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_code.items():
                text_surface = my_font_num.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            screen.blit(arrow, arrow_rect)
            screen.blit(deletea, deletea_rect)

            pygame.display.flip()

            for event in pygame.event.get():
                if(event.type is MOUSEBUTTONDOWN):
                    pos = pygame.mouse.get_pos()
                elif(event.type is MOUSEBUTTONUP):
                    pos = pygame.mouse.get_pos()
                    x,y = pos

            if x < 180 and y > 60 and enter_num < 4:
                if y > 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "1"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "2"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "3"

                if y > 120 and y < 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "4"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "5"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "6"

                if y > 60 and y < 120:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "7"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "8"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "9"

                enter_num = enter_num + 1
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 120 and y < 180 and enter_num < 4:
                my_code[(30 + enter_num * 60,30)] = "0"
                enter_num = enter_num + 1
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 60 and y < 120 and enter_num > 0:
                enter_num = enter_num - 1
                my_code[(30 + enter_num * 60,30)] = ""
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 180:
                str_code = my_code[(30,30)] + my_code[(90,30)] + my_code[(150,30)] + my_code[(210,30)]
                print(ID)
                check_result = checkpw(ID, str_code)
                print(check_result)
                if check_result == 1:
                    my_code[(30,30)] = ''
                    my_code[(90,30)] = ''
                    my_code[(150,30)] = ''
                    my_code[(210,30)] = ''
                    enter_num = 0
                    status_flag = 3
                x = 0
                y = 0
            if x > 240:
                my_code[(30,30)] = ''
                my_code[(90,30)] = ''
                my_code[(150,30)] = ''
                my_code[(210,30)] = ''
                enter_num = 0
                status_flag = 0
                x = 0
                y = 0

        time_start=time.time()
        while status_flag == 3:
            s = "Welcome"
            str_out = s1 + s + s3
            os.system(str_out)
            #sendm()
            screen.fill(BLACK)
            for text_pos, my_text in my_warning2.items():
                text_surface = my_font_big.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            pygame.display.flip()
            opend()
            time.sleep(3)
            iii = 0
            while GPIO.input(26):
                iii = iii + 1
                print(iii)
            closed()
            status_flag = 0
            #time_now = time.time()
            #time_interval = time_now - time_start
            #if time_interval > 10:
            #    status_flag = 0

        speak_flag = 0
        while status_flag == 4:
            if speak_flag == 0:
                s = "Hello, please enter password to register"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)
            screen.fill(BLACK)
            pygame.draw.rect(screen,WHITE,[240,0,80,240],0)
            for text_pos, my_text in my_number.items():
                text_surface = my_font_num.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_main_button.items():
                text_surface = my_font_num.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_code.items():
                text_surface = my_font_num.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            screen.blit(arrow, arrow_rect)
            screen.blit(deletea, deletea_rect)

            pygame.display.flip()

            for event in pygame.event.get():
                if(event.type is MOUSEBUTTONDOWN):
                    pos = pygame.mouse.get_pos()
                elif(event.type is MOUSEBUTTONUP):
                    pos = pygame.mouse.get_pos()
                    x,y = pos

            if x < 180 and y > 60 and enter_num < 4:
                if y > 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "1"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "2"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "3"

                if y > 120 and y < 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "4"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "5"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "6"

                if y > 60 and y < 120:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "7"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "8"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "9"

                enter_num = enter_num + 1
                x = 0
                y = 0
            if x > 180 and x < 240 and y > 120 and y < 180 and enter_num < 4:
                my_code[(30 + enter_num * 60,30)] = "0"
                enter_num = enter_num + 1
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 60 and y < 120 and enter_num > 0:
                enter_num = enter_num - 1
                my_code[(30 + enter_num * 60,30)] = ""
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 180:
                str_code = my_code[(30,30)] + my_code[(90,30)] + my_code[(150,30)] + my_code[(210,30)]
                print(ID)
                check_result = checkpw(ID, str_code)
                print(check_result)
                if check_result == 1:
                    my_code[(30,30)] = ''
                    my_code[(90,30)] = ''
                    my_code[(150,30)] = ''
                    my_code[(210,30)] = ''
                    enter_num = 0
                    status_flag = 5
                x = 0
                y = 0
            if x > 240:
                my_code[(30,30)] = ''
                my_code[(90,30)] = ''
                my_code[(150,30)] = ''
                my_code[(210,30)] = ''
                enter_num = 0
                status_flag = 0
                x = 0
                y = 0

        speak_flag = 0
        while status_flag == 5:
            if speak_flag == 0:
                s = "please look at the camera to help me gather information about you"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)
            screen.fill(BLACK)
            #showing :  strating collecting info here
            pygame.display.flip()
            gather_info_success_flag = 0
            gather_info_success_flag = gather_info()
            if gather_info_success_flag:
                s = "information gathered successfully, please wait"
                str_out = s1 + s + s3
                os.system(str_out)
                screen.fill(BLACK)
                #showing :  training now, please wait
                for text_pos, my_text in my_warning1.items():
                    text_surface = my_font_num.render(my_text, True, WHITE)
                    rect = text_surface.get_rect(center=text_pos)
                    screen.blit(text_surface, rect)

                pygame.display.flip()
                train()
                print("train out")
                gather_info_success_flag = 0
                status_flag = 0

        speak_flag = 0
        while status_flag == 6:
            if speak_flag == 0:
                s = "please enter your old password"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)
            screen.fill(WHITE)
            pygame.draw.rect(screen,BLACK,[240,0,80,240],0)
            for text_pos, my_text in my_number.items():
                text_surface = my_font_num.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_main_button.items():
                text_surface = my_font_num.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_code.items():
                text_surface = my_font_num.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            screen.blit(arrow, arrow_rect)
            screen.blit(deletea, deletea_rect)

            pygame.display.flip()

            for event in pygame.event.get():
                if(event.type is MOUSEBUTTONDOWN):
                    pos = pygame.mouse.get_pos()
                elif(event.type is MOUSEBUTTONUP):
                    pos = pygame.mouse.get_pos()
                    x,y = pos

            if x < 180 and y > 60 and enter_num < 4:
                if y > 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "1"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "2"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "3"

                if y > 120 and y < 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "4"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "5"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "6"

                if y > 60 and y < 120:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "7"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "8"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "9"

                enter_num = enter_num + 1
                x = 0
                y = 0
            if x > 180 and x < 240 and y > 120 and y < 180 and enter_num < 4:
                my_code[(30 + enter_num * 60,30)] = "0"
                enter_num = enter_num + 1
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 60 and y < 120 and enter_num > 0:
                enter_num = enter_num - 1
                my_code[(30 + enter_num * 60,30)] = ""
                x = 0
                y = 0
            if x > 180 and x < 240 and y > 180:
                str_code = my_code[(30,30)] + my_code[(90,30)] + my_code[(150,30)] + my_code[(210,30)]
                print(ID)
                check_result = checkpw(ID, str_code)
                print(check_result)
                if check_result == 1:
                    s = "correct password"
                    str_out = s1 + s + s3
                    os.system(str_out)
                    my_code[(30,30)] = ''
                    my_code[(90,30)] = ''
                    my_code[(150,30)] = ''
                    my_code[(210,30)] = ''
                    enter_num = 0
                    status_flag = 7
                x = 0
                y = 0
            if x > 240:
                my_code[(30,30)] = ''
                my_code[(90,30)] = ''
                my_code[(150,30)] = ''
                my_code[(210,30)] = ''
                enter_num = 0
                status_flag = 0
                x = 0
                y = 0

        speak_flag = 0
        while status_flag == 7:
            if speak_flag == 0:
                s = "please enter your new password"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)
            screen.fill(WHITE)
            pygame.draw.rect(screen,BLACK,[240,0,80,240],0)
            for text_pos, my_text in my_number.items():
                text_surface = my_font_num.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_main_button.items():
                text_surface = my_font_num.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            for text_pos, my_text in my_code.items():
                text_surface = my_font_num.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            screen.blit(arrow, arrow_rect)
            screen.blit(deletea, deletea_rect)

            pygame.display.flip()

            for event in pygame.event.get():
                if(event.type is MOUSEBUTTONDOWN):
                    pos = pygame.mouse.get_pos()
                elif(event.type is MOUSEBUTTONUP):
                    pos = pygame.mouse.get_pos()
                    x,y = pos

            if x < 180 and y > 60 and enter_num < 4:
                if y > 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "1"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "2"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "3"

                if y > 120 and y < 180:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "4"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "5"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "6"

                if y > 60 and y < 120:
                    if x < 60:
                        my_code[(30 + enter_num * 60,30)] = "7"
                    if x > 60 and x < 120:
                        my_code[(30 + enter_num * 60,30)] = "8"
                    if x > 120:
                        my_code[(30 + enter_num * 60,30)] = "9"

                enter_num = enter_num + 1
                x = 0
                y = 0
            if x > 180 and x < 240 and y > 120 and y < 180 and enter_num < 4:
                my_code[(30 + enter_num * 60,30)] = "0"
                enter_num = enter_num + 1
                x = 0
                y = 0

            if x > 180 and x < 240 and y > 60 and y < 120 and enter_num > 0:
                enter_num = enter_num - 1
                my_code[(30 + enter_num * 60,30)] = ""
                x = 0
                y = 0
            if x > 180 and x < 240 and y > 180:
                str_code = my_code[(30,30)] + my_code[(90,30)] + my_code[(150,30)] + my_code[(210,30)]
                if my_code[(30,30)] != "" and my_code[(90,30)] != "" and my_code[(150,30)] != "" and my_code[(210,30)] != "":
                #print(ID)
                #check_result = checkpw(ID, str_code)
                #print(check_result)
                #if check_result == 1:
                    my_code[(30,30)] = ''
                    my_code[(90,30)] = ''
                    my_code[(150,30)] = ''
                    my_code[(210,30)] = ''
                    changepw(str_code)
                    s = "password reset succeed"
                    str_out = s1 + s + s3
                    os.system(str_out)
                    enter_num = 0
                    status_flag = 0

                x = 0
                y = 0
            if x > 240:
                my_code[(30,30)] = ''
                my_code[(90,30)] = ''
                my_code[(150,30)] = ''
                my_code[(210,30)] = ''
                enter_num = 0
                status_flag = 0
                x = 0
                y = 0

        speak_flag == 0
        while status_flag == 8:
            if speak_flag == 0:
                s = "E-mail has been sent to the owner, waiting for response now"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)
            screen.fill(BLACK)
            for text_pos, my_text in my_warning4.items():
                text_surface = my_font_info.render(my_text, True, WHITE)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
            pygame.display.flip()
            #$start_web = time.time()
            sendm()
            camera_flag = 3
            app.run(host="0.0.0.0", threaded=True)

        while status_flag == 9:
            sub_flag = 1
            while sub_flag:
                screen.fill(BLACK)
                for text_pos, my_text in my_warning4.items():
                    text_surface = my_font_info.render(my_text, True, WHITE)
                    rect = text_surface.get_rect(center=text_pos)
                    screen.blit(text_surface, rect)
                pygame.display.flip()

        speak_flag == 0
        while status_flag == 10:
            if speak_flag == 0:
                s = "Use wechat to login"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)

            screen.fill(WHITE)
            ii = 0
            for text_pos, my_text in my_message1.items():
                text_surface = my_font_info.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            pygame.display.flip()
            #itchat.get_QR(picDir='./qr.png')
            itchat.login(screen = screen, picDir = './qr.png')
            #qr = pygame.image.load('./qr.png')
            #qr = pygame.transform.scale(qr,(200,200))
            #screen.blit(qr,(120,40))
            #itchat.login(picDir = './qr.png')

            for text_pos, my_text in my_message2.items():
                text_surface = my_font_info.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)

            pygame.display.flip()
            camera.capture('./realtime.jpg')
            itchat.send('Let that guy get in? [yes/no]', toUserName = 'filehelper')
            itchat.send_image('./realtime.jpg',toUserName = 'filehelper')

            for text_pos, my_text in my_message3.items():
                text_surface = my_font_info.render(my_text, True, BLACK)
                rect = text_surface.get_rect(center=text_pos)
                screen.blit(text_surface, rect)
                #ii = ii + 1

            pygame.display.flip()
            itchat.start_receiving()
            itchat.run()

        speak_flag = 0
        if status_flag == 11:
            timea = time.time()
        while status_flag == 11:
            if speak_flag == 0:
                s = "Voice recognition part porbiddened due to sound card conflict, sorry"
                speak_flag = 1
                str_out = s1 + s + s3
                os.system(str_out)
            screen.fill(WHITE)
            screen.blit(mic, mic_rect)
            pygame.display.flip()
            with open("/home/pi/final_project/voiceorder.txt") as file:
                data1=file.read()
                file.close()
                ti = time.time()-timea
                if ti >10:
                    status_flag = 0
                    break
                #print (data1)
                if data1 == 'open':
                    print('^_^')
                    status_flag = 3
                    break
            #i=i+1

except KeyboardInterrupt:
    pass

p_left.stop()
p_right.stop()
GPIO.cleanup()


text:
from __future__ import print_function

import numpy as np
import sys
from logging import warning
from struct import pack
import matplotlib.pyplot as plt

def display_profiles(gamma, m, pos, u, ax):
    r = np.linalg.norm(pos, axis=1)
    hist, edges = np.histogram(r, bins='auto')
    nbins = len(hist)
    rho_c = np.zeros(nbins)
    u_c   = np.zeros(nbins)
    r_c   = np.zeros(nbins)
    for i in range(len(hist)):
        vol      = 4 * np.pi / 3. * (edges[i+1]**3 - edges[i]**3)
        rho_c[i] = m * hist[i] / vol
        r_c[i]   = (edges[i+1] + edges[i]) / 2.
        ig = np.where((r < edges[i+1]) & (r > edges[i]))
        u_c[i] = np.mean(u[ig[0]])

    ax1 = ax[0]
    ax2 = ax[1]
    ax1.minorticks_on()
    ax2.minorticks_on()
    ax1.plot(r_c, rho_c, 'go', label='Stretched particles')
    ax1.legend(loc='best', numpoints=1, title=r'$\gamma=$%.1f'%(gamma))
    plt.setp(ax1.get_xticklabels(), visible=False)
    ax1.set_ylabel(r"$\rho$")
    ax2.plot(r_c, u_c, 'go', label='Stretched particles')
    ax2.set_xlabel(r"$r$")
    ax2.set_ylabel(r"$u$")

def save_particles(ids, pos, vel, mass, u, outfile, format):

    N = len(pos)

    if format == 0:
        # Openning file
        try:
            ofile = open(outfile,'w')
        except IOError as e:
            msg = "IO Error({0}): {1}".format(e.errno, e.strerror)
            warning(msg)
        except:
            print("Unexpected error: {}".format(sys.exc_info()[0]))
            raise

        id_space = len("{}".format(N))

        # Preparing every line to print to the file
        for i in range(N):
            # Formatting particle attributes
            ie = '% d' % ids[i]
            me = '% 3.8e' % mass[i]
            rx = '% 3.8e' % pos[i][0]
            ry = '% 3.8e' % pos[i][1]
            rz = '% 3.8e' % pos[i][2]
            vx = '% 3.8e' % pos[i][0]
            vy = '% 3.8e' % pos[i][1]
            vz = '% 3.8e' % pos[i][2]
            ue = '% 3.8e' % u[i]

            # Right-align the strings
            outstring = "{0} {1} {2} {3} {4} {5} {6} {7} {8}\n".\
                         format( ie.rjust(id_space),\
                                 me.rjust(12),\
                                 rx.rjust(12),\
                                 ry.rjust(12),\
                                 rz.rjust(12),\
                                 vx.rjust(12),\
                                 vy.rjust(12),\
                                 vz.rjust(12),\
                                 ue.rjust(12))
            # Write to file
            ofile.write(outstring)

        # Closing the file
        ofile.close()

    elif format == 1:
        Ngas = len(mass)
        Npart = np.array([Ngas, 0, 0, 0, 0, 0])
        Nmass = np.array([0, 0, 0, 0, 0, 0])
        # Linearizing the 3D-array of the position and velocity
        pos = pos.ravel()
        vel = vel.ravel()

        dummy = np.zeros(Npart[0])
        time = 0.
        redshift = 0.0  # double
        flag_sfr = 0  # long
        flag_feedback = 0  # long
        bytesleft = 256 - 6*4 - 6*8 - 8 - 8 - 2*4 - 6*4
        fill = np.zeros(int(bytesleft/4.0), dtype=np.int)  # int

        with open(outfile, 'wb') as f:
            nbytes = 256
            # Header
            f.write(pack('i', nbytes))
            f.write(pack('i' * len(Npart), *Npart))
            f.write(pack('d' * len(Nmass), *Nmass))
            f.write(pack('d', time))
            f.write(pack('d', redshift))
            f.write(pack('i', flag_sfr))
            f.write(pack('i', flag_feedback))
            f.write(pack('i' * len(Npart), *Npart))
            f.write(pack('i' * len(fill), *fill))
            f.write(pack('i', nbytes))

            # Positions
            nbytes = int(len(pos) * 4)
            f.write(pack('i', nbytes))
            f.write(pack('f' * len(pos), *pos))
            f.write(pack('i', nbytes))

            # Velocities
            f.write(pack('i', nbytes))
            f.write(pack('f' * len(vel), *vel))
            f.write(pack('i', nbytes))

            # Ids
            nbytes = int(len(ids) * 4)
            f.write(pack('i', nbytes))
            f.write(pack('i' * len(ids), *ids))
            f.write(pack('i', nbytes))

            # Masses
            nbytes = len(mass) * 4
            f.write(pack('i', nbytes))
            f.write(pack('f' * len(mass), *mass))
            f.write(pack('i', nbytes))

            # Energy
            nbytes = len(u) * 4
            f.write(pack('i', nbytes))
            f.write(pack('f' * len(u), *u))
            f.write(pack('i', nbytes))

max_stars_repo_name: fgoicovic/gadget-polytrope

max_issues_repo_name: fgoicovic/gadget-polytrope

max_forks_repo_name: fgoicovic/gadget-polytrope


text:
from sklearn.metrics import *
import matplotlib.pyplot as plt
import os
import pandas as pd
import seaborn as sns
import numpy as np

class ModelMetrics:
    def __init__(self,
                 y_true: list,
                 y_pred_scores: list,
                 threshold=0.5,
                 y_pred=None,
                 model=None,
                 save_dir=None):

        self.y_true = np.array(y_true)
        self.y_pred_scores = np.array(y_pred_scores)
        self.threshold = threshold
        self.model = model
        self.save_dir = save_dir if save_dir is not None else "./"

        if y_pred is None:
            self.y_pred = self.get_y_pred()
        else:
            self.y_pred = y_pred

        self.lw = 1.3

    def get_roc_curve(self,
                      figsize: tuple = (8, 8),
                      show: bool = False):
        """
        :return: auc score of receiver operating characteristic curve
        """

        fpr, tpr, threshold = roc_curve(self.y_true, self.y_pred_scores)

        plt.figure(figsize=figsize)
        plt.plot(fpr,
                 tpr,
                 'k-',
                 lw=self.lw,
                 label='ROC AUC Score (area = %0.2f)' % auc(fpr, tpr))

        plt.plot([0, 1], [0, 1], color='black', linewidth=self.lw, linestyle='-.')
        plt.xlim([-0.005, 1.0])
        plt.ylim([0, 1.05])
        plt.xlabel('False Positive Rate', fontsize=13)
        plt.ylabel('True Positive Rate', fontsize=13)
        plt.title('ROC curve')
        plt.legend(loc="lower right")

        if self.save_dir is not None:
            plt.savefig(os.path.join(self.save_dir, "roc_curve.jpg"), bbox_inches='tight')

        if show:
            plt.show()
            plt.close()

        return dict(
            fpr=fpr,
            tpr=tpr,
            thresholds=threshold,
            auc=auc(fpr, tpr)
        )

    def get_pr_curve(self,
                     figsize: tuple = (8, 8),
                     show: bool = False):
        """
        :return: auc score of precision-recall curve
        """
        precision, recall, thresholds = precision_recall_curve(self.y_true, self.y_pred_scores)
        plt.figure(figsize=figsize)
        plt.plot(recall,
                 precision,
                 'k-',
                 lw=self.lw,
                 label='PR AUC Score (area = %0.2f)' % auc(recall, precision))

        plt.xlim([-0.005, 1.0])
        plt.ylim([0, 1.05])
        plt.xlabel('Recall', fontsize=13)
        plt.ylabel('Precision', fontsize=13)
        plt.title('Precision-Recall curve')
        plt.legend(loc="lower right")

        if self.save_dir is not None:
            plt.savefig(os.path.join(self.save_dir, "pr_curve.jpg"), bbox_inches='tight')

        if show:
            plt.show()
            plt.close()

        return dict(
            precision=precision,
            recall=recall,
            thresholds=thresholds,
            auc=auc(recall, precision)
        )

    def get_y_pred(self):
        if type(self.y_pred_scores) != np.ndarray:
            y_pred_scores = np.array(self.y_pred_scores)
            y_pred = np.where(y_pred_scores > self.threshold, 1, 0)
        else:
            y_pred = np.where(self.y_pred_scores > self.threshold, 1, 0)
        return y_pred

    def get_confusion_matrix(self,
                             show: bool = True,
                             positive_class_name: str = "TCFA > 200",
                             negative_class_name: str = "Negative"):
        conf_matrix = confusion_matrix(self.y_true, self.y_pred)

        if show:
            df_cm = pd.DataFrame(conf_matrix,
                                 index=[i for i in [negative_class_name, positive_class_name]],
                                 columns=[i for i in [negative_class_name, positive_class_name]])
            plt.figure(figsize=(10, 8))
            sns.heatmap(df_cm, annot=True)

            if self.save_dir is not None:
                plt.savefig(os.path.join(self.save_dir, "confusion_matrix.jpg"),
                            bbox_inches='tight')

            # plt.show()
            plt.close()

        return conf_matrix

    def get_ppv_npv(self) -> tuple:
        conf_matrix = confusion_matrix(self.y_true, self.y_pred)

        tp = conf_matrix[1, 1]
        tn = conf_matrix[0, 0]
        fp = conf_matrix[0, 1]
        fn = conf_matrix[1, 0]

        ppv = tp / (tp + fp)
        npv = tn / (tn + fn)

        return ppv, npv

    def get_sen_spe(self) -> tuple:
        """
            Get Sensitivity and Specificity
        :return:
            sensitivity, specificity
        """
        conf_matrix = confusion_matrix(self.y_true, self.y_pred)

        tp = conf_matrix[1, 1]
        tn = conf_matrix[0, 0]
        fp = conf_matrix[0, 1]
        fn = conf_matrix[1, 0]

        sen = tp / (tp + fn)
        spe = tn / (tn + fp)

        return sen, spe

    def get_f1_score(self):
        return f1_score(self.y_true, self.y_pred, average='micro', pos_label=1)

    def get_precision_score(self):
        conf_matrix = confusion_matrix(self.y_true, self.y_pred)

        tp = conf_matrix[1, 1]
        fp = conf_matrix[0, 1]

        precision = tp / (tp + fp)
        return precision

    def get_accuracy_score(self):
        return accuracy_score(self.y_true, self.y_pred)

    def get_recall_score(self):
        conf_matrix = confusion_matrix(self.y_true, self.y_pred)

        tp = conf_matrix[1, 1]
        fn = conf_matrix[1, 0]

        recall = tp / (tp + fn)
        return recall
        # return recall_score(self.y_true, self.y_pred, average='micro', pos_label=1)

    def print_all(self):

        _recall_score = self.get_recall_score()
        _precision_score = self.get_precision_score()
        conf_matrix = self.get_confusion_matrix()
        ro_auc_score = self.get_roc_curve()['auc']
        pr_auc_score = self.get_pr_curve()['auc']
        ppv, npv = self.get_ppv_npv()
        sensitivity, specificity = self.get_sen_spe()
        accuracy = self.get_accuracy_score()

        print(f"Precision score : {_precision_score}")
        print(f"Recall score : {_recall_score}")
        print(f"ROC-AUC score : {ro_auc_score}")
        print(f"PR-AUC score : {pr_auc_score}")
        print(f"Accuracy score : {accuracy}")
        print(f"PPV : {ppv}")
        print(f"NPV : {npv}")
        print(f"Sensitivity : {sensitivity}")
        print(f"Specificity : {specificity}")

        result = {
            "recall": _recall_score,
            "precision": _precision_score,
            "roc-auc": ro_auc_score,
            "pr-auc": pr_auc_score,
            "ppv": ppv,
            "npv": npv,
            "accuracy": accuracy,
            "sensitivity": sensitivity,
            "specificity": specificity,
            "confusion_matrix": conf_matrix
        }

        pd.DataFrame.from_dict(result, orient="index").to_csv(os.path.join(self.save_dir, "result.csv"))
        return result

if __name__ == '__main__':
    pass

max_stars_repo_name: SWKoreaBME/MLP-Mixer-PyTorch

max_issues_repo_name: SWKoreaBME/MLP-Mixer-PyTorch

max_forks_repo_name: SWKoreaBME/MLP-Mixer-PyTorch


text:
import os
import unittest

import numpy as np
import xarray as xr

from xcube.api.gen.config import get_config_dict
from xcube.api.gen.gen import gen_cube
from xcube.util.dsio import rimraf
from .helpers import get_inputdata_path

def clean_up():
    files = ['l2c-single.nc', 'l2c.nc', 'l2c.zarr', 'l2c-single.zarr']
    for file in files:
        rimraf(file)
        rimraf(file + '.temp.nc')  # May remain from Netcdf4DatasetIO.append()
    rimraf(get_inputdata_path("input.txt"))

class DefaultProcessTest(unittest.TestCase):

    def setUp(self):
        clean_up()

    def tearDown(self):
        clean_up()

    # noinspection PyMethodMayBeStatic
    def test_process_inputs_single(self):
        status = gen_cube_wrapper(
            [get_inputdata_path('20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c-single.nc',
            False
        )
        self.assertEqual(True, status)

    def test_process_inputs_append_multiple_nc(self):
        status = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c.nc',
            True
        )
        self.assertEqual(True, status)

    def test_process_inputs_append_multiple_zarr(self):
        status = gen_cube_wrapper(
            [get_inputdata_path('201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc')],
            'l2c.zarr',
            True
        )
        self.assertEqual(True, status)

    def test_input_txt(self):
        f = open((os.path.join(os.path.dirname(__file__), 'inputdata', "input.txt")), "w+")
        for i in range(1, 4):
            file_name = "2017010" + str(i) + "-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc"
            file = get_inputdata_path(file_name)
            f.write("%s\n" % file)
        f.close()
        status = gen_cube_wrapper(
            [get_inputdata_path('input.txt')],
            'l2c.zarr',
            True
        )
        self.assertEqual(True, status)

    def test_handle_360_lon(self):
        status = gen_cube_wrapper(
            [get_inputdata_path('20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc')],
            'l2c-single.zarr',
            True
        )
        self.assertEqual(True, status)
        ds = xr.open_zarr('l2c-single.zarr')
        self.assertIn('lon', ds.coords)
        self.assertFalse(np.any(ds.coords['lon'] > 180.))

# noinspection PyShadowingBuiltins
def gen_cube_wrapper(input_paths, output_path, append_mode):
    config = get_config_dict(locals())
    return gen_cube(input_processor='default',
                    output_size=(320, 180),
                    output_region=(-4., 47., 12., 56.),
                    output_resampling='Nearest',
                    output_variables=[('analysed_sst', dict(name='SST'))],
                    dry_run=False,
                    monitor=None,
                    **config)

max_stars_repo_name: dzelge/xcube

max_issues_repo_name: dzelge/xcube

max_forks_repo_name: dzelge/xcube