text:
from __future__ import absolute_import
import numpy as np
import sys

from .speech_manip import spline_smooth_fzero

# gaussian type smoothing, convolution with hamming window
def _smooth(params, win, type="HAMMING"):

    win = int(win+0.5)
    if win >= len(params)-1:
        win = len(params)-1
    if win % 2 == 0:
        win+=1

    s = np.r_[params[win-1:0:-1],params,params[-1:-win:-1]]

    if type=="HAMMING":
        w = np.hamming(win)
        third = int(win/5)
        #w[:third] = 0
    else:
        w = np.ones(win)

    y = np.convolve(w/w.sum(),s,mode='valid')
    return y[(win/2):-(win/2)]

def _interpolate_zeros(params, method='pchip', min_val = 0):
    import scipy.stats
    import scipy.interpolate
    voiced = np.array(params, float)
    voiced[voiced<=min_val] = np.nan

    if np.isnan(voiced[-1]):
        voiced[-1] = np.nanmin(voiced)
    if np.isnan(voiced[0]):
        voiced[0] = scipy.stats.nanmean(voiced)

    not_nan = np.logical_not(np.isnan(voiced))
    indices = np.arange(len(voiced))
    if method == 'spline':
        interp = scipy.interpolate.UnivariateSpline(indices[not_nan],voiced[not_nan], k=2) #, s=0.5)
        # return voiced parts intact
        smoothed = interp(indices)
        for i in range(0, len(smoothed)):
            if params[i] > min_val:
                smoothed[i] = params[i]
        return smoothed
    elif method =='pchip':

        interp = scipy.interpolate.pchip(indices[not_nan], voiced[not_nan])
    else:
        interp = scipy.interpolate.interp1d(indices[not_nan], voiced[not_nan], method)
    return interp(indices)

# true envelope style smoothing with varying window
def _peak_smooth(params, max_iter, win,min_win=2,voicing=[]):

    smooth=np.array(params)
    win_reduce =  np.exp(np.linspace(np.log(win),np.log(min_win), max_iter))

    TRACE = False
    if TRACE:
        pylab.plot(params, 'black')
    for i in range(0,max_iter):

        smooth = np.maximum(params,smooth)
        if TRACE:
            if i> 0 and i % 10 == 0:
                pylab.plot(smooth,'gray',linewidth=1)

        if len(voicing) >0:
            smooth = _smooth(smooth,int(win+0.5)) #,type='rectangle')
            smooth[voicing>0] = params[voicing>0]
        else:
            smooth = _smooth(smooth,int(win+0.5))#,type='rectangle')

        win = win_reduce[i]

    if TRACE:
        pylab.plot(smooth,'red',linewidth=2)
        raw_input()

    return smooth

def process(filename, param_type=None, voicing = [], n_coeff = 1):

    param_type = param_type.lower()
    try:
        param = np.loadtxt(filename)
    except:
        from scipy.io import wavfile
        fs, param = wavfile.read(sys.argv[1]+".wav")

    processed = np.array(param)

    # for f0, handle boundaries, remove outliers and interpolate zeros
    if param_type =="f0_antti":
        TRACE = True
        fix = True
        if TRACE:
            import pylab
            pylab.plot(param)
        voiced = np.nonzero(param)
        # 1) set start silence to median and end silence to min
        first_v=voiced[0][0]
        last_v = voiced[0][-1]
        processed[:first_v] = np.median(param[voiced])
        processed[last_v:]= np.min(param[voiced])
        processed[param==0] = np.min(processed[voiced])

        if fix:

            # 2) remove gross outliers

            ## 2a) true envelope style smoothing with varying (rectangular) window
            processed = _peak_smooth(processed, 200,10, voicing=param)
            smooth = _smooth(processed, 100)
            param[abs(processed/smooth)>1.3] = 0
            processed[abs(processed/smooth)>1.3] = np.min(processed)
            processed = _peak_smooth(processed, 200,20, voicing=param)

            # constrain speed of change
            diff = np.array(processed)
            diff[1:] = np.diff(processed)
            diff[1:] = _smooth(diff[1:], 5)
            processed = np.cumsum(diff)

            # update unvoiced defualts
            processed[:first_v] = np.median(processed)
            processed[last_v:]= np.min(processed)
            processed[param==0] = np.min(processed)

        #fill unvoiced gaps
        processed = _peak_smooth(processed, 200,20, voicing=param)

        #smooth transitions
        processed = _peak_smooth(processed, 10,5)

        if TRACE:
            pylab.plot(processed,'black',linewidth=2)

        if TRACE:
            raw_input()

    # for f0, handle boundaries, remove outliers and interpolate zeros
    if param_type =="f0":
        TRACE = False
        if TRACE:
            import pylab
            pylab.plot(param)
        processed = spline_smooth_fzero(param, trim_n_frames=4, s=200, k=1)
        if TRACE:
            pylab.plot(processed,'black',linewidth=2)
        if TRACE:
            raw_input()
            pylab.clf()

    # for gain, remove log, find envelope that fills unvoiced parts
    elif param_type=="gain":
        import scipy.signal
        processed = scipy.signal.medfilt(param, 3)
        processed =  np.e**(processed*0.1)

        processed = _peak_smooth(processed,100,8,voicing=voicing)

    return processed

def duration(labels):
    """
    make a continuous duration signal from time labels
    x-axis = time
    y-axis = log-duration
    """

    TRACE = False
    if TRACE:
        import pylab
        pylab.figure()

    params = np.zeros(labels[-1][1])

    for i in range(len(labels)):
        (st,en,phn) = labels[i]
        params[int(st+(en-st)/2.0)] = np.log(en-st)

    # fix gaps

    for i in range(len(labels)-1):
        cur = labels[i][1]
        next = labels[i+1][0]
        if cur != next:
            params[int(cur+(next-cur)/2.0)] = 1 #np.log(next-cur)

    if labels[0][0] != 0:
        params[labels[0][0]/2.0] = 1 #np.log(labels[0][0]/2.0)

    params = _interpolate_zeros(params,'pchip') #,'linear')

    if TRACE:
        pylab.plot(params, label="interpolated",linewidth=2)
        #cwt_utils.plot_labels(labels,ypos=-2)
        pylab.plot(params)
        raw_input()

    # for boundaries, look at derivative, i.e. change from slow ta fast
    #params[:-1] = np.diff(params,1)
    #params[-1] = params[-2]

    if TRACE:
        pylab.plot(params*30, label="difference",linewidth=2)
        pylab.legend()
        raw_input()

    return params

max_stars_repo_name: cadia-lvl/Ossian

max_issues_repo_name: cadia-lvl/Ossian

max_forks_repo_name: cadia-lvl/Ossian


text:
"""
Author: Xu Yuecong
"""
import os
import cv2
import numpy as np

import torch.utils.data as data
import logging

from .flow_iterator import Flow
from .video_iterator import Video

class DualIter(data.Dataset):

    def __init__(self,
                 video_prefix,
                 flow_prefix,
                 txt_list,
                 video_sampler,
                 flow_sampler,
                 video_transforms=None,
                 flow_transforms=None,
                 name="<NO_NAME>",
                 force_color=True,
                 force_gray=True,
                 cached_info_path=None,
                 return_item_subpath=False,
                 shuffle_list_seed=None,
                 check_video=False,
                 tolerant_corrupted_video=None):
        super(DualIter, self).__init__()
        # load params
        self.video_sampler = video_sampler
        self.flow_sampler = flow_sampler
        self.force_color = force_color
        self.force_gray = force_gray
        self.video_prefix = video_prefix
        self.flow_prefix = flow_prefix
        self.video_prefix_x = os.path.join(flow_prefix, 'flow_x')
        self.video_prefix_y = os.path.join(flow_prefix, 'flow_y')
        self.video_transforms = video_transforms
        self.flow_transforms = flow_transforms
        self.return_item_subpath = return_item_subpath
        self.backup_item = None
        if (not check_video) and (tolerant_corrupted_video is None):
            logging.warning("DualIter:: >> `check_video' is off, `tolerant_corrupted_video' is automatically activated.")
            tolerant_corrupted_video = True
        self.tolerant_corrupted_video = tolerant_corrupted_video
        self.rng = np.random.RandomState(shuffle_list_seed if shuffle_list_seed else 0)
        # load video list
        self.video_list = self._get_video_list(video_prefix=self.video_prefix,
                                               txt_list=txt_list,
                                               check_video=check_video,
                                               cached_info_path=cached_info_path)
        if shuffle_list_seed is not None:
            self.rng.shuffle(self.video_list)
        logging.info("DualIter:: iterator initialized (phase: '{:s}', num: {:d})".format(name, len(self.video_list)))

    def getitem_from_raw_video(self, index):
        # get current video info
        v_id, label, vid_subpath, frame_count = self.video_list[index]
        video_path = os.path.join(self.video_prefix, vid_subpath)
        video_path_x = os.path.join(self.video_prefix_x, vid_subpath)
        video_path_y = os.path.join(self.video_prefix_y, vid_subpath)

        faulty_frames = []
        successfule_trial = False
        try:
            with Video(vid_path=video_path) as video:
                if frame_count < 0:
                    frame_count = video.count_frames(check_validity=False)
                for i_trial in range(20):
                    # dynamic sampling
                    sampled_idxs = self.video_sampler.sampling(range_max=frame_count, v_id=v_id, prev_failed=(i_trial>0))
                    if not 'SegmentalSampling' in self.video_sampler.__class__.__name__:
                        sampled_idxs = list(sampled_idxs)
                    if set(sampled_idxs).intersection(faulty_frames):
                        continue
                    prev_sampled_idxs = sampled_idxs
                    # extracting frames
                    sampled_frames = video.extract_frames(idxs=sampled_idxs, force_color=self.force_color)
                    if sampled_frames is None:
                        faulty_frames.append(video.faulty_frame)
                    else:
                        successfule_trial = True
                        break
            with Flow(vid_path=video_path_x) as video_x:
                if frame_count < 0:
                    frame_count = video_x.count_frames(check_validity=False)
                for i_trial in range(20):
                    # dynamic sampling
                    sampled_idxs_x = self.flow_sampler.sampling(range_max=frame_count, v_id=v_id, prev_failed=(i_trial>0))
                    if not 'SegmentalSampling' in self.flow_sampler.__class__.__name__:
                        sampled_idxs_x = list(sampled_idxs_x)
                    if set(sampled_idxs_x).intersection(faulty_frames):
                        continue
                    prev_sampled_idxs = sampled_idxs_x
                    # extracting frames
                    sampled_frames_x = video_x.extract_frames(idxs=sampled_idxs_x, force_gray=self.force_gray)
                    if sampled_frames_x is None:
                        faulty_frames.append(video_x.faulty_frame)
                    else:
                        successfule_trial = True
                        with Flow(vid_path=video_path_y) as video_y:
                            sampled_frames_y = video_y.extract_frames(idxs=sampled_idxs_x, force_gray=self.force_gray)
                        break
        except IOError as e:
            logging.warning(">> I/O error({0}): {1}".format(e.errno, e.strerror))

        if not successfule_trial:
            assert (self.backup_item is not None), \
                "DualIter:: >> frame {} is error & backup is inavailable. [{}]'".format(faulty_frames, video_path_x)
            # logging.warning(">> frame {} is error, use backup item! [{}]".format(faulty_frames, video_path_x))
            with Video(vid_path=self.backup_item['video_path']) as video:
                sampled_frames = video.extract_frames(idxs=self.backup_item['sampled_idxs'], force_color=self.force_color)
            with Flow(vid_path=self.backup_item['video_path_x']) as video_x:
                sampled_frames_x = video_x.extract_frames(idxs=self.backup_item['sampled_idxs_x'], force_gray=self.force_gray)
            with Flow(vid_path=self.backup_item['video_path_y']) as video_y:
                sampled_frames_y = video_y.extract_frames(idxs=self.backup_item['sampled_idxs_y'], force_gray=self.force_gray)
        elif self.tolerant_corrupted_video:
            # assume the error rate less than 10%
            if (self.backup_item is None) or (self.rng.rand() < 0.1):
                self.backup_item = {'video_path': video_path, 'video_path_x': video_path_x, 'video_path_y': video_path_y,
                                    'sampled_idxs': sampled_idxs, 'sampled_idxs_x': sampled_idxs_x, 'sampled_idxs_y': sampled_idxs_x}

        sampled_frames_flow = []
        for idx in range(len(sampled_frames_x)):
            sampled_frames_flow.append(np.stack([sampled_frames_x[idx], sampled_frames_y[idx]], axis=2))
        clip_input = np.concatenate(sampled_frames, axis=2)
        clip_input_flow = np.concatenate(sampled_frames_flow, axis=2)
        # apply video augmentation
        if self.video_transforms is not None:
            clip_input = self.video_transforms(clip_input)
        if self.flow_transforms is not None:
            clip_input_flow = self.flow_transforms(clip_input_flow)
        return clip_input, clip_input_flow, label, vid_subpath

    def __getitem__(self, index):
        succ = False
        attempts = 0
        while not succ and attempts < 5:
            try:
                clip_input, clip_input_flow, label, vid_subpath = self.getitem_from_raw_video(index)
                succ = True
            except Exception as e:
                index = self.rng.choice(range(0, self.__len__()))
                attempts = attempts + 1
                logging.warning("DualIter:: ERROR!! (Force using another index:\n{})\n{}".format(index, e))

        if self.return_item_subpath:
            return clip_input, clip_input_flow, label, vid_subpath
        else:
            return clip_input, clip_input_flow, label

    def __len__(self):
        return len(self.video_list)

    def _get_video_list(self,
                        video_prefix,
                        txt_list,
                        check_video=False,
                        cached_info_path=None):
        # formate:
        # [vid, label, video_subpath, frame_count]
        assert os.path.exists(video_prefix), "DualIter:: failed to locate: `{}'".format(video_prefix)
        assert os.path.exists(txt_list), "DualIter:: failed to locate: `{}'".format(txt_list)

        # try to load cached list
        cached_video_info = {}
        if cached_info_path:
            if os.path.exists(cached_info_path):
                f = open(cached_info_path, 'r')
                cached_video_prefix = f.readline().split()[1]
                cached_txt_list = f.readline().split()[1]
                if (cached_video_prefix == video_prefix.replace(" ", "")) \
                    and (cached_txt_list == txt_list.replace(" ", "")):
                    logging.info("DualIter:: loading cached video info from: `{}'".format(cached_info_path))
                    lines = f.readlines()
                    for line in lines:
                        video_subpath, frame_count = line.split()
                        cached_video_info.update({video_subpath: int(frame_count)})
                else:
                    logging.warning(">> Cached video list mismatched: " +
                                    "(prefix:{}, list:{}) != expected (prefix:{}, list:{})".format(\
                                    cached_video_prefix, cached_txt_list, video_prefix, txt_list))
                f.close()
            else:
                if not os.path.exists(os.path.dirname(cached_info_path)):
                    os.makedirs(os.path.dirname(cached_info_path))

        # building dataset
        video_list = []
        new_video_info = {}
        logging_interval = 100
        with open(txt_list) as f:
            lines = f.read().splitlines()
            logging.info("DualIter:: found {} videos in `{}'".format(len(lines), txt_list))
            for i, line in enumerate(lines):
                v_id, label, video_subpath = line.split()
                video_path_x = os.path.join(video_prefix, video_subpath)
                if not os.path.exists(video_path_x):
                    # logging.warning("DualIter:: >> cannot locate `{}'".format(video_path_x))
                    continue
                if check_video:
                    if video_subpath in cached_video_info:
                        frame_count = cached_video_info[video_subpath]
                    elif video_subpath in new_video_info:
                        frame_count = cached_video_info[video_subpath]
                    else:
                        frame_count = self.video.open(video_path_x).count_frames(check_validity=True)
                        new_video_info.update({video_subpath: frame_count})
                else:
                    frame_count = -1
                info = [int(v_id), int(label), video_subpath, frame_count]
                video_list.append(info)
                if check_video and (i % logging_interval) == 0:
                    logging.info("DualIter:: - Checking: {:d}/{:d}, \tinfo: {}".format(i, len(lines), info))

        # caching video list
        if cached_info_path and len(new_video_info) > 0:
            logging.info("DualIter:: adding {} lines new video info to: {}".format(len(new_video_info), cached_info_path))
            cached_video_info.update(new_video_info)
            with open(cached_info_path, 'w') as f:
                # head
                f.write("video_prefix: {:s}\n".format(video_prefix.replace(" ", "")))
                f.write("txt_list: {:s}\n".format(txt_list.replace(" ", "")))
                # content
                for i, (video_subpath, frame_count) in enumerate(cached_video_info.items()):
                    if i > 0:
                        f.write("\n")
                    f.write("{:s}\t{:d}".format(video_subpath, frame_count))

        return video_list

ext: py

reason: import numpy


text:
#-*- coding: utf-8 -*-
import sys
import traceback
import pyocr
import pyocr.builders
from PIL import Image
import numpy as np
import copy
import os

######################################tesseract工具##########################################################
# 1、生成gif文件
# 2、转化为tif，是convert命令
# 3、tesseract douban.font.exp0.tif douban.font.exp0 batch.nochop makebox生成box文件，再使用jTe工具打开编辑调整
# 4、touch  执行改批处理前先要目录下创建font_properties文件，内容为font 0 0 0 0
# 5、tesseract  douban.font.exp0.tif douban.font.exp0 nobatch box.train
# 6、unicharset_extractor  douban.font.exp0.box
# 6、mftraining -F font_properties -U unicharset -O douban.unicharset douban.font.exp0.tr
# 7、cntraining  douban.font.exp0.tr
# 8、mv normproto douban.normproto
# mv inttemp douban.inttemp
# mv pffmtable douban.pffmtable
# mv shapetable douban.shapetable
# 9、combine_tessdata douban
############################################################################################################

WHITE = (0, 0, 0)
BLACK = (255, 255, 255)

cwddir = "/home/allen/downloadfiles/gittest/DoubanAuto/example"
imagdir = "/home/allen/downloadfiles/gittest/DoubanAuto/image"
trfimagedir = "/home/allen/downloadfiles/gittest/DoubanAuto/image/tfrimg-src"
trftifdir = "/home/allen/downloadfiles/gittest/DoubanAuto/image/tfrimg-tif"
rmnosietifdir = "/home/allen/downloadfiles/gittest/DoubanAuto/image/rmnosi-img-tif"

def batchConvertTif(trfimagedir):

    time = os.popen('date')
    print time.read()

    for path, dir, filename in os.walk(trfimagedir):
        for f in filename:
            status = os.popen('convert %s %s' % (path + '/' +f, path + '/' + f + '.tif'))
            print status.read()

def deal_each_img(path,filename,rsfname):

    if rsfname == '' or rsfname == None:
        rsfname = filename

    im = Image.open(path + '/' + filename)
    imb = copy.deepcopy(im)
    gray_img(im)
    interference_point(im, "11")
    remove_noise(im)
    if os.path.exists(rmnosietifdir) == 0:
        os.mkdir(rmnosietifdir)

    im.save(rmnosietifdir + '/' + rsfname)
    return rmnosietifdir + '/' + rsfname

def test_deal_img(path, filename, rsfname):

    im = Image.open(path+'/'+filename)
    imb = copy.deepcopy(im)
    # im.show()

    # im_gray = im.convert("L")
    gray_img(im)
    # im.show()
    # 转换成灰度图像
    # im_gray = imb.convert("L")
    # im_gray.show()
    # #
    # # 保存图像
    # im_gray.save(cwddir + "image_gray.jpg")

    interference_point(im, "11")
    # im.show()

    remove_noise(im)
    # im.show()
    if os.path.exists(trfimagedir) == 0:
        os.mkdir(trfimagedir)

    im.save(trfimagedir + '/' + rsfname)

    return trfimagedir + '/' + rsfname

# 灰度化并且二值化图片
def gray_img(_img):
    img1 = _img.load()

    print "after load()",   type(img1)
    print "jpg:",   type(_img)

    # print _img.shape
    print _img.size
    rows, cols = _img.size

    threshold = 23
    for i in range(rows):
        for j in range(cols):
            r, g, b = _img.getpixel((i, j))
            if r > threshold or g > threshold or b > threshold:
                img1[i, j] = (255, 255, 255)
            else:
                img1[i, j] = (0, 0, 0)

def int(t):
    r, g, b = t
    return r*256*256 + g*256 + b

# 点降噪
def interference_point(_img,img_name, x = 0, y = 0):

    img = _img.load()
    """
    9邻域框,以当前点为中心的田字框,黑点个数
    :param x:
    :param y:
    :return:
    """
    # filename =  './out_img/' + img_name.split('.')[0] + '-interferencePoint.jpg'
    # todo 判断图片的长宽度下限

    cur_pixel = img[x, y]# 当前像素点的值
    height, width = _img.size
    for y in range(0, width - 1):
      for x in range(0, height - 1):
        if y == 0:  # 第一行
            if x == 0:  # 左上顶点,4邻域
                # 中心点旁边3个点
                sum = int(cur_pixel) \
                      + int(img[x, y + 1]) \
                      + int(img[x + 1, y]) \
                      + int(img[x + 1, y + 1])
                if sum <= 2 * 245:
                  img[x, y] = 0
            elif x == height - 1:  # 右上顶点
                sum = int(cur_pixel) \
                      + int(img[x, y + 1]) \
                      + int(img[x - 1, y]) \
                      + int(img[x - 1, y + 1])
                if sum <= 2 * 245:
                  img[x, y] = 0
            else:  # 最上非顶点,6邻域
                sum = int(img[x - 1, y]) \
                      + int(img[x - 1, y + 1]) \
                      + int(cur_pixel) \
                      + int(img[x, y + 1]) \
                      + int(img[x + 1, y]) \
                      + int(img[x + 1, y + 1])
                if sum <= 3 * 245:
                  img[x, y] = 0
        elif y == width - 1:  # 最下面一行
            if x == 0:  # 左下顶点
                # 中心点旁边3个点
                sum = int(cur_pixel) \
                      + int(img[x + 1, y]) \
                      + int(img[x + 1, y - 1]) \
                      + int(img[x, y - 1])
                if sum <= 2 * 245:
                  img[x, y] = 0
            elif x == height - 1:  # 右下顶点
                sum = int(cur_pixel) \
                      + int(img[x, y - 1]) \
                      + int(img[x - 1, y]) \
                      + int(img[x - 1, y - 1])
                if sum <= 2 * 245:
                  img[x, y] = 0
            else:  # 最下非顶点,6邻域
                sum = int(cur_pixel) \
                      + int(img[x - 1, y]) \
                      + int(img[x + 1, y]) \
                      + int(img[x, y - 1]) \
                      + int(img[x - 1, y - 1]) \
                      + int(img[x + 1, y - 1])
                if sum <= 3 * 245:
                  img[x, y] = 0
        else:  # y不在边界
            if x == 0:  # 左边非顶点
                sum = int(img[x, y - 1]) \
                      + int(cur_pixel) \
                      + int(img[x, y + 1]) \
                      + int(img[x + 1, y - 1]) \
                      + int(img[x + 1, y]) \
                      + int(img[x + 1, y + 1])
                if sum <= 3 * 245:
                  img[x, y] = 0
            elif x == height - 1:  # 右边非顶点
                sum = int(img[x, y - 1]) \
                      + int(cur_pixel) \
                      + int(img[x, y + 1]) \
                      + int(img[x - 1, y - 1]) \
                      + int(img[x - 1, y]) \
                      + int(img[x - 1, y + 1])
                if sum <= 3 * 245:
                  img[x, y] = 0
            else:  # 具备9领域条件的
                sum = int(img[x - 1, y - 1]) \
                      + int(img[x - 1, y]) \
                      + int(img[x - 1, y + 1]) \
                      + int(img[x, y - 1]) \
                      + int(cur_pixel) \
                      + int(img[x, y + 1]) \
                      + int(img[x + 1, y - 1]) \
                      + int(img[x + 1, y]) \
                      + int(img[x + 1, y + 1])
                if sum <= 4 * 245:
                  img[x, y] = 0
    # cv2.imwrite(filename,img)
    return img

def remove_noise(_img, window=1):
    """ 中值滤波移除噪点
        """
    img = _img.load()
    if window == 1:
        # 十字窗口
        window_x = [1, 0, 0, -1, 0]
        window_y = [0, 1, 0, 0, -1]
    elif window == 2:
        # 3*3矩形窗口
        window_x = [-1, 0, 1, -1, 0, 1, 1, -1, 0]
        window_y = [-1, -1, -1, 1, 1, 1, 0, 0, 0]

    width, height = _img.size

    for i in xrange(width):
        for j in xrange(height):
            box = []
            black_count, white_count = 0, 0
            for k in xrange(len(window_x)):
                d_x = i + window_x[k]
                d_y = j + window_y[k]
                try:
                    d_point = img[d_x, d_y]
                    if d_point == BLACK:
                        box.append(1)
                    else:
                        box.append(0)
                except IndexError:
                    img[i, j] = WHITE
                    continue

            box.sort()
            if len(box) == len(window_x):
                mid = box[len(box) / 2]
                if mid == 1:
                    img[i, j] = BLACK
                else:
                    img[i, j] = WHITE

#按照阀值设置灰度
def dealImg(self):
    width, heith = self.img.size
    threshold = 21
    for i in range(0, width):
        for j in range(0, heith):
            p = self.frame[i, j]
            r, g, b = p
            if r > threshold or g > threshold or b > threshold:
                self.frame[i, j] = WHITE
            else:
                self.frame[i, j] = BLACK

def batch_trasf(path1 ):

    index = 0
    for path, dirs, files in os.walk(path1):
        for file in files:
                index = index + 1
                print file, path
                ## 处理每一个文件
                test_deal_img(path, file, "%s.jpg" % str(index))

def test():
    try:
        tools = pyocr.get_available_tools()
        if len(tools) == 0:
            print("No OCR tool found")
            sys.exit(1)
        else:
            for k in tools:
                print ("The tools list are  '%s' " % str(k.get_name()))

        # The tools are returned in the recommended order of usage
        tool = tools[0]
        print("Will use tool '%s'" % (tool.get_name()))
        # Ex: Will use tool 'libtesseract'
        langs = tool.get_available_languages()
        print("Available languages: %s" % ", ".join(langs))
        lang = langs[2]
        print("Will use lang '%s'" % (lang))
        # Ex: Will use lang 'fra'
        # Note that languages are NOT sorted in any way. Please refer
        # to the system locale settings for the default language
        # to use.
    except:
        traceback.print_exc()

def test_ocr(imagepath):

    tool = pyocr.get_available_tools()[0]
    #buider_ = pyocr.builders.TextBuilder()
    lang = tool.get_available_languages()[2]
    txt = tool.image_to_string(Image.open('/home/allen/downloadfiles/gittest/DoubanAuto/image/1.jpg'), lang=lang)
    #txt = tool.image_to_string(Image.open('/home/allen/downloadfiles/gittest/DoubanAuto/util/11.png'), lang=lang)
    print txt

def ocrImg(img_path, filename):

    #减噪并且灰度化
    tobeOcr_path_file = deal_each_img(img_path, filename, filename)

    tool = pyocr.get_available_tools()[0]
    lang = tool.get_available_languages()[2]
    txt = tool.image_to_string(Image.open(tobeOcr_path_file), lang=lang)

    print txt
    return txt

if __name__ == '__main__':
    # test_ocr("")
    test()
# test()

max_stars_repo_name: Dijkstr-LJ/DoubanAuto

max_issues_repo_name: Dijkstr-LJ/DoubanAuto

max_forks_repo_name: Dijkstr-LJ/DoubanAuto


text:
import pygame
import numpy as np

from .utils import Color, ChessImages, RED_SIDE, BLUE_SIDE

class ChessPiece:
    NOT_SELECTED = 0
    SELECTED = 1

    NAME = "A chess piece"

    def __init__(self, centrePoint=(0, 0), position=(0, 0), side=RED_SIDE):
        self.position = position
        self.centrePoint = centrePoint
        self.radius = 20

        self.attackingPiece = False
        self.side = side
        self.status = self.NOT_SELECTED
        self.possibleMoves = []
        self.image = None

    def _getImage(self):
        """
        Get image of piece depends on its side and type
        """
        if isinstance(self, Horse):
            return (
                ChessImages.RED_HORSE
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_HORSE
            )

        if isinstance(self, Soldier):
            return (
                ChessImages.RED_SOLDIER
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_SOLDIER
            )

        if isinstance(self, Elephant):
            return (
                ChessImages.RED_ELEPHANT
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_ELEPHANT
            )

        if isinstance(self, Chariot):
            return (
                ChessImages.RED_CHARIOT
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_CHARIOT
            )

        if isinstance(self, Cannon):
            return (
                ChessImages.RED_CANNON
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_CANNON
            )

        if isinstance(self, Advisor):
            return (
                ChessImages.RED_ADVISOR
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_ADVISOR
            )

        if isinstance(self, Lord):
            return (
                ChessImages.RED_LORD
                if self.getSide() == RED_SIDE
                else ChessImages.BLUE_LORD
            )

    def draw(self, win):
        """
        Draw the piece
        """
        pygame.draw.circle(win, Color.WHITE, self.centrePoint, self.radius)
        x, y = self.centrePoint
        if self.status == self.SELECTED:
            pygame.draw.rect(
                win,
                Color.GREEN,
                pygame.Rect(
                    x - self.radius - 1,
                    y - self.radius - 1,
                    self.radius * 2 + 2,
                    self.radius * 2 + 2,
                ),
                2,
            )

        image = self._getImage()
        win.blit(image, (x - self.radius, y - self.radius))

    def isClicked(self, pos=None):
        """
        Check if the piece is clicked
        """
        if not pos:
            return None

        clickX, clickY = pos
        centerX, centerY = self.centrePoint

        if (clickX - centerX) ** 2 + (clickY - centerY) ** 2 < self.radius ** 2:
            return True
        return False

    def getPosition(self):
        return self.position

    def makeSelected(self):
        """
        Make the piece seleceted
        """
        self.status = self.SELECTED

    def deselect(self):
        """
        Deleselect the piece
        """
        self.status = self.NOT_SELECTED

    def checkPossibleMove(self, boardGrid):
        """
        Checking all possible moves of the piece
        """
        print("This is not a real chesspiece")

    def checkForAttackAbility(self):
        """
        Depends on the piece's position, determine whether they are a threat to the other lord or not
        """
        return None

    def moveToNewSpot(self, centrePoint=None, position=None):
        """
        Change the piece attribute according to the new spot
        """
        if not centrePoint:
            return

        self.centrePoint = centrePoint
        self.position = position

        self.checkForAttackAbility()

    def getSide(self):
        return self.side

    def isEnemy(self, other):
        return True if other.getSide() != self.getSide() else False

    def __str__(self):
        return f"A {self.NAME} at {self.position}"

    def __repr__(self):
        return f"Object {self.NAME}-{self.side}"

    def __eq__(self, other):
        if not isinstance(other, ChessPiece):
            return False
        return self.centrePoint == other.centrePoint

    def __hash__(self):
        return hash(self.centrePoint)

class Chariot(ChessPiece):
    """
    Chariot chess piece
    """

    NAME = "Chariot"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.attackingPiece = True

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        column = boardGrid[:, colPos]  # get the whole colummn
        # Move up
        for r in range(
            rowPos - 1, -1, -1
        ):  # get all the row above current row, except current row
            if not column[r]:
                movables.append((r, colPos))
            else:
                otherPiece = column[r]
                if self.isEnemy(otherPiece):
                    movables.append((r, colPos))
                break

        # Move down
        for r in range(rowPos + 1, 10):
            if not column[r]:
                movables.append((r, colPos))
            else:
                otherPiece = column[r]
                if self.isEnemy(otherPiece):
                    movables.append((r, colPos))
                break

        row = boardGrid[rowPos]  # get the whole row
        # Move left
        for c in range(colPos - 1, -1, -1):
            if not row[c]:
                movables.append((rowPos, c))
            else:
                otherPiece = row[c]
                if self.isEnemy(otherPiece):
                    movables.append((rowPos, c))
                break

        # Move right
        for c in range(colPos + 1, 9):
            if not row[c]:
                movables.append((rowPos, c))
            else:
                otherPiece = row[c]
                if self.isEnemy(otherPiece):
                    movables.append((rowPos, c))
                break

        if update:
            self.possibleMoves = movables

        return movables

class Cannon(ChessPiece):
    """
    Cannon chess piece
    """

    NAME = "Cannon"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.attackingPiece = True

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        column = boardGrid[:, colPos]  # get the whole colummn
        skip = False
        # Move up
        for r in range(
            rowPos - 1, -1, -1
        ):  # get all the row above current row, except current row
            if not skip:
                if column[r] is None:
                    movables.append((r, colPos))
                else:
                    skip = True
            else:
                otherPiece = column[r]
                if otherPiece is not None:
                    if self.isEnemy(otherPiece):
                        movables.append((r, colPos))
                    break

        # Move down
        skip = False
        for r in range(rowPos + 1, 10):
            if not skip:
                if column[r] is None:
                    movables.append((r, colPos))
                else:
                    skip = True
            else:
                otherPiece = column[r]
                if otherPiece is not None:
                    if self.isEnemy(otherPiece):
                        movables.append((r, colPos))
                    break

        row = boardGrid[rowPos]  # get the whole row
        # Move left
        skip = False
        for c in range(colPos - 1, -1, -1):
            if not skip:
                if row[c] is None:
                    movables.append((rowPos, c))
                else:
                    skip = True
            else:
                otherPiece = row[c]
                if otherPiece is not None:
                    if self.isEnemy(otherPiece):
                        movables.append((rowPos, c))
                    break

        # Move right
        skip = False
        for c in range(colPos + 1, 9):
            if not skip:
                if row[c] is None:
                    movables.append((rowPos, c))
                else:
                    skip = True
            else:
                otherPiece = row[c]
                if otherPiece is not None:
                    if self.isEnemy(otherPiece):
                        movables.append((rowPos, c))
                    break

        if update:
            self.possibleMoves = movables

        return movables

class Horse(ChessPiece):
    """
    Horse Chess Piece
    """

    NAME = "Horse"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        # All possibles move of a Knight
        moveX = []
        moveY = []

        # Move up
        stepX = [-2, -2]
        stepY = [-1, 1]
        if 0 <= rowPos - 1 and not boardGrid[rowPos - 1, colPos]:
            moveX += stepX
            moveY += stepY

        # Move down
        stepX = [2, 2]
        stepY = [-1, 1]
        if rowPos + 1 <= 9 and not boardGrid[rowPos + 1, colPos]:
            moveX += stepX
            moveY += stepY

        # Move left
        stepX = [1, -1]
        stepY = [-2, -2]
        if 0 <= colPos - 1 and not boardGrid[rowPos, colPos - 1]:
            moveX += stepX
            moveY += stepY

        # Move right
        stepX = [1, -1]
        stepY = [2, 2]
        if colPos + 1 <= 8 and not boardGrid[rowPos, colPos + 1]:
            moveX += stepX
            moveY += stepY

        for mX, mY in zip(moveX, moveY):
            newRow = rowPos + mX
            newCol = colPos + mY

            # Check if the move is valid
            if 0 <= newRow <= 9 and 0 <= newCol <= 8:
                if not boardGrid[newRow, newCol]:
                    movables.append((newRow, newCol))
                else:
                    otherPiece = boardGrid[newRow, newCol]
                    if self.isEnemy(otherPiece):
                        movables.append((newRow, newCol))

        if update:
            self.possibleMoves = movables

        return movables

    def checkForAttackAbility(self):
        if self.side == BLUE_SIDE and self.position[0] > 4:
            self.attackingPiece = True

        if self.side == RED_SIDE and self.position[0] < 5:
            self.attackingPiece = True

class Elephant(ChessPiece):
    """
    Horse Chess Piece
    """

    NAME = "Elephant"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.moveLimit = (0, 4) if 0 <= self.position[0] <= 4 else (5, 9)

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        upLimit, downLimt = self.moveLimit

        # Move up left
        if (
            rowPos - 1 >= upLimit
            and colPos - 1 >= 0
            and not boardGrid[rowPos - 1, colPos - 1]
        ):
            if rowPos - 2 >= upLimit and colPos - 2 >= 0:
                if not boardGrid[rowPos - 2, colPos - 2]:
                    movables.append((rowPos - 2, colPos - 2))
                else:
                    otherPiece = boardGrid[rowPos - 2, colPos - 2]
                    if self.isEnemy(otherPiece):
                        movables.append((rowPos - 2, colPos - 2))

        # Move up right
        if (
            rowPos - 1 >= upLimit
            and colPos + 1 <= 8
            and not boardGrid[rowPos - 1, colPos + 1]
        ):
            if rowPos - 2 >= upLimit and colPos + 2 <= 8:
                if not boardGrid[rowPos - 2, colPos + 2]:
                    movables.append((rowPos - 2, colPos + 2))
                else:
                    otherPiece = boardGrid[rowPos - 2, colPos + 2]
                    if self.isEnemy(otherPiece):
                        movables.append((rowPos - 2, colPos + 2))

        # Move down left
        if (
            rowPos + 1 <= downLimt
            and colPos - 1 >= 0
            and not boardGrid[rowPos + 1, colPos - 1]
        ):
            if rowPos + 2 >= upLimit and colPos - 2 >= 0:
                if not boardGrid[rowPos + 2, colPos - 2]:
                    movables.append((rowPos + 2, colPos - 2))
                else:
                    otherPiece = boardGrid[rowPos + 2, colPos - 2]
                    if self.isEnemy(otherPiece):
                        movables.append((rowPos + 2, colPos - 2))

        # Move down right
        if (
            rowPos + 1 <= downLimt
            and colPos + 1 <= 8
            and not boardGrid[rowPos + 1, colPos + 1]
        ):
            if rowPos + 2 >= upLimit and colPos + 2 <= 8:
                if not boardGrid[rowPos + 2, colPos + 2]:
                    movables.append((rowPos + 2, colPos + 2))
                else:
                    otherPiece = boardGrid[rowPos + 2, colPos + 2]
                    if self.isEnemy(otherPiece):
                        movables.append((rowPos + 2, colPos + 2))

        if update:
            self.possibleMoves = movables

        return movables

class Soldier(ChessPiece):
    NAME = "Soldier"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.goSideWay = False
        self.riverLine = 4 if self.position[0] < 4 else 5
        self.direction = 1 if self.position[0] < 4 else -1

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        if self.direction == 1 and rowPos > self.riverLine:
            self.goSideWay = True

            if rowPos > self.riverLine + 1:
                self.attackingPiece = True

        if self.direction == -1 and rowPos < self.riverLine:
            self.goSideWay = True

            if rowPos < self.riverLine - 1:
                self.attackingPiece = True

        # Move up
        newRow = rowPos + self.direction
        if 0 <= newRow <= 9:
            if boardGrid[newRow, colPos] is None or self.isEnemy(
                boardGrid[newRow, colPos]
            ):
                movables.append((newRow, colPos))

        if self.goSideWay:
            directionsX = [1, -1]
            for direction in directionsX:
                newCol = colPos + direction
                if 0 <= newCol <= 8:
                    if boardGrid[rowPos, newCol] is None or self.isEnemy(
                        boardGrid[rowPos, newCol]
                    ):
                        movables.append((rowPos, newCol))

        if update:
            self.possibleMoves = movables

        return movables

class Lord(ChessPiece):
    NAME = "Lord"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.moveLimit = (0, 2) if 0 <= self.position[0] <= 2 else (7, 9)
        self.mated = False

        # Animation when lord is under attack
        self.grow = True
        self.thickness = 4
        self.skip = True

    def draw(self, win):
        super().draw(win)
        if self.mated:
            if self.grow:
                self.thickness += 1
            else:
                self.thickness -= 1

            if self.thickness == 6:
                self.grow = False
            if self.thickness <= 4:
                self.grow = True

            pygame.draw.circle(
                win,
                Color.RED,
                self.centrePoint,
                self.radius + self.thickness,
                self.thickness,
            )

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        upLimit, downLimt = self.moveLimit
        leftLimit, rightLimit = (3, 5)

        movesX = [0, 0, 1, -1]
        movesY = [-1, 1, 0, 0]

        for mX, mY in zip(movesX, movesY):
            newRow = rowPos + mX
            newCol = colPos + mY

            # Check if the move is valid
            if upLimit <= newRow <= downLimt and leftLimit <= newCol <= rightLimit:
                if not boardGrid[newRow, newCol]:
                    movables.append((newRow, newCol))
                else:
                    otherPiece = boardGrid[newRow, newCol]
                    if self.isEnemy(otherPiece):
                        movables.append((newRow, newCol))

        if update:
            self.possibleMoves = movables

        return movables

class Advisor(ChessPiece):
    NAME = "Advisor"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.moveLimit = (0, 2) if 0 <= self.position[0] <= 2 else (7, 9)

    def checkPossibleMove(self, boardGrid, update=True):
        boardGrid = np.array(boardGrid)
        movables = []

        rowPos, colPos = self.position

        upLimit, downLimt = self.moveLimit
        leftLimit, rightLimit = (3, 5)

        movesX = [1, 1, -1, -1]
        movesY = [-1, 1, 1, -1]

        for mX, mY in zip(movesX, movesY):
            newRow = rowPos + mX
            newCol = colPos + mY

            # Check if the move is valid
            if upLimit <= newRow <= downLimt and leftLimit <= newCol <= rightLimit:
                if not boardGrid[newRow, newCol]:
                    movables.append((newRow, newCol))
                else:
                    otherPiece = boardGrid[newRow, newCol]
                    if self.isEnemy(otherPiece):
                        movables.append((newRow, newCol))

        if update:
            self.possibleMoves = movables

        return movables


text:
#!/usr/bin/env python3
import unittest

import numpy as np
import numpy.testing as nptest
import pandas as pd
import pandas.testing as pdtest

from datafold.utils.general import *

class TestDataStructureUtils(unittest.TestCase):
    def setUp(self) -> None:
        self._create_random_series()
        self._create_random_dataframe()

    def _create_random_series(self):
        self.series1 = pd.Series(np.random.rand(10), index=np.arange(10))

    def _create_random_dataframe(self):
        self.df1 = pd.DataFrame(
            np.random.rand(10, 3), index=np.arange(10), columns=["A", "B", "C"]
        )
        self.df2 = pd.DataFrame(
            np.random.rand(10, 1), index=np.arange(10), columns=["A"]
        )

    def test_series_if_applicable1(self):
        actual = series_if_applicable(self.series1)
        pdtest.assert_series_equal(actual, self.series1)

    def test_series_if_applicable2(self):
        actual = series_if_applicable(self.df1)
        pdtest.assert_frame_equal(actual, self.df1)

    def test_series_if_applicable3(self):
        actual = series_if_applicable(self.df2)
        pdtest.assert_series_equal(actual, self.df2.iloc[:, 0])

    def test_is_df_same_index_columns1(self):
        with self.assertRaises(AssertionError):
            is_df_same_index(self.df1, self.df2, handle="raise")

    def test_is_df_same_index_columns2(self):
        df1_local = self.df1.copy()
        is_df_same_index(self.df1, df1_local, handle="raise")

    def test_is_df_same_index_columns3(self):
        df1_local = self.df1.copy()
        df1_local.columns = ["One", "Two", "Three"]

        with self.assertRaises(AssertionError):
            is_df_same_index(self.df1, df1_local)

    def test_is_integer1(self):
        self.assertTrue(is_integer(6.0))

    def test_is_integer2(self):
        self.assertTrue(is_integer(np.float_(5)))

    def test_is_integer3(self):
        self.assertFalse(is_integer(1.3))

    def test_is_integer4(self):
        self.assertFalse(is_integer(np.array([1, 2])))

    def test_is_float1(self):
        self.assertTrue(is_float(1.0))

    def test_is_float2(self):
        self.assertFalse(is_float(1))

    def test_is_float3(self):
        self.assertTrue(is_float(np.float16(1.0)))

    def test_is_float4(self):
        self.assertFalse(is_float(np.int8(1.0)))

    def test_if1dim_colvec1(self):
        actual = if1dim_colvec(np.array([1, 2, 3]))

        self.assertIsInstance(actual, np.ndarray)
        self.assertEqual(actual.ndim, 2)
        self.assertEqual(actual.shape[0], 3)

    def test_if1dim_colvec2(self):
        vec = np.array([1, 2, 3])[np.newaxis, :]
        actual = if1dim_colvec(vec)

        nptest.assert_array_equal(actual, vec)

    def test_if1dim_colvec3(self):
        vec = np.array([1, 2, 3])[:, np.newaxis]
        actual = if1dim_colvec(vec)

        nptest.assert_array_equal(actual, vec)

    def test_if1dim_rowvec1(self):
        actual = if1dim_rowvec(np.array([1, 2, 3]))

        self.assertIsInstance(actual, np.ndarray)
        self.assertEqual(actual.ndim, 2)
        self.assertEqual(actual.shape[1], 3)

    def test_if1dim_rowvec2(self):
        vec = np.array([1, 2, 3])[:, np.newaxis]
        actual = if1dim_colvec(vec)
        nptest.assert_array_equal(actual, vec)

    def test_if1dim_rowvec3(self):
        vec = np.array([1, 2, 3])[np.newaxis, :]
        actual = if1dim_colvec(vec)
        nptest.assert_array_equal(actual, vec)

# if __name__ == "__main__":
#     TestDataStructureUtils().test_if1dim_colvec2()
#     actual = if1dim_rowvec(np.array([1, 2, 3]))


text:
def smooth_signal(x,window_len=10,window='flat'):

    """smooth the data using a window with requested size.

    This method is based on the convolution of a scaled window with the signal.
    The signal is prepared by introducing reflected copies of the signal
    (with the window size) in both ends so that transient parts are minimized
    in the begining and end part of the output signal.
    The code taken from: https://scipy-cookbook.readthedocs.io/items/SignalSmooth.html

    input:
        x: the input signal
        window_len: the dimension of the smoothing window; should be an odd integer
        window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
                'flat' window will produce a moving average smoothing.

    output:
        the smoothed signal
    """

    if x.ndim != 1:
        raise(ValueError, "smooth only accepts 1 dimension arrays.")

    if x.size < window_len:
        raise(ValueError, "Input vector needs to be bigger than window size.")

    if window_len<3:
        return x

    if window_len % 2 == 1:
      window_len -= 1

    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
        raise(ValueError, "Window is one of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")

    s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]

    if window == 'flat': # Moving average
        w=np.ones(window_len,'d')
    else:
        w=eval('np.'+window+'(window_len)')

    y=np.convolve(w/w.sum(),s,mode='valid')

    return y[(int(window_len/2)-1):-int(window_len/2)]

from scipy.signal import butter, filtfilt

def butter_lowpass(cutoff, fs, order=10):
    nyq = 0.5 * fs
    normal_cutoff = cutoff / nyq
    b, a = butter(order, normal_cutoff, btype='low', analog=False)
    return b, a

def butter_lowpass_filter(data, cutoff, fs, order=10):
    b, a = butter_lowpass(cutoff, fs, order=order)
    y = filtfilt(b, a, data)
    return y

def flatten_signal(signal,lambda_=1e4,order=0.5,itermax=50):

# Find the signal baseline using airPLS alghorithm
  # add_beginning=600
  # signal = np.insert(signal, 0, signal[0]*np.ones(add_beginning)).reshape(len(signal)+add_beginning,1)
  # base = airPLS(signal[:,0],lambda_=lambda_,porder=order,itermax=itermax).reshape(len(signal),1)
  # signal = signal[add_beginning:]
  # base = base[add_beginning:]

  add=600
  s = np.r_[signal[add-1:0:-1],signal,signal[-2:-add-1:-1]]
  b = airPLS(s,lambda_=lambda_,porder=order,itermax=itermax)
  signal = s[add-1:-add+1]
  base = b[add-1:-add+1]

# Remove the begining of the signal baseline
  signal = (signal - base)

  return signal, base

def standardize_signal(signal):

  z_signal = (signal - np.nanmedian(signal)) / np.nanstd(signal)

  return z_signal

'''
airPLS.py Copyright 2014 Renato Lombardo - renato.lombardo@unipa.it
Baseline correction using adaptive iteratively reweighted penalized least squares

This program is a translation in python of the R source code of airPLS version 2.0
by Yizeng Liang and Zhang Zhimin - https://code.google.com/p/airpls
Reference:
Z.-M. Zhang, S. Chen, and Y.-Z. Liang, Baseline correction using adaptive iteratively reweighted penalized least squares. Analyst 135 (5), 1138-1146 (2010).

Description from the original documentation:

Baseline drift always blurs or even swamps signals and deteriorates analytical results, particularly in multivariate analysis.  It is necessary to correct baseline drift to perform further data analysis. Simple or modified polynomial fitting has been found to be effective in some extent. However, this method requires user intervention and prone to variability especially in low signal-to-noise ratio environments. The proposed adaptive iteratively reweighted Penalized Least Squares (airPLS) algorithm doesn't require any user intervention and prior information, such as detected peaks. It iteratively changes weights of sum squares errors (SSE) between the fitted baseline and original signals, and the weights of SSE are obtained adaptively using between previously fitted baseline and original signals. This baseline estimator is general, fast and flexible in fitting baseline.

LICENCE
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>
'''

import numpy as np
from scipy.sparse import csc_matrix, eye, diags
from scipy.sparse.linalg import spsolve

def WhittakerSmooth(x,w,lambda_,differences=1):
    '''
    Penalized least squares algorithm for background fitting

    input
        x: input data (i.e. chromatogram of spectrum)
        w: binary masks (value of the mask is zero if a point belongs to peaks and one otherwise)
        lambda_: parameter that can be adjusted by user. The larger lambda is,  the smoother the resulting background
        differences: integer indicating the order of the difference of penalties

    output
        the fitted background vector
    '''
    X=np.matrix(x)
    m=X.size
    #i=np.arange(0,m)
    E=eye(m,format='csc')
    D=E[1:]-E[:-1] # numpy.diff() does not work with sparse matrix. This is a workaround.
    W=diags(w,0,shape=(m,m))
    A=csc_matrix(W+(lambda_*D.T*D))
    B=csc_matrix(W*X.T)
    background=spsolve(A,B)
    return np.array(background)

def airPLS(x, lambda_=100, porder=1, itermax=15):
    '''
    Adaptive iteratively reweighted penalized least squares for baseline fitting

    input
        x: input data (i.e. chromatogram of spectrum)
        lambda_: parameter that can be adjusted by user. The larger lambda is,  the smoother the resulting background, z
        porder: adaptive iteratively reweighted penalized least squares for baseline fitting

    output
        the fitted background vector
    '''
    m=x.shape[0]
    w=np.ones(m)
    for i in range(1,itermax+1):
        z=WhittakerSmooth(x,w,lambda_, porder)
        d=x-z
        dssn=np.abs(d[d<0].sum())
        if(dssn<0.001*(abs(x)).sum() or i==itermax):
            if(i==itermax): print('WARING max iteration reached!')
            break
        w[d>=0]=0 # d>0 means that this point is part of a peak, so its weight is set to 0 in order to ignore it
        w[d<0]=np.exp(i*np.abs(d[d<0])/dssn)
        w[0]=np.exp(i*(d[d<0]).max()/dssn)
        w[-1]=w[0]
    return z

def interpolate_signal(signal,t_old,t_new):

  from scipy.interpolate import interp1d

  func = interp1d(t_old,signal)
  s_new = func(t_new)

  return s_new

def fit_signal(signal, reference, model='RANSAC'):

  i0 = np.max(np.argwhere(np.isnan(signal)))
  signal = signal[i0+1:]
  reference = reference[i0+1:]

  signal = np.array(signal).reshape(len(signal),1)
  reference = np.array(reference).reshape(len(reference),1)

# Positive linear regression
  if model == 'RANSAC':
    from sklearn.linear_model import RANSACRegressor
    lin = RANSACRegressor(max_trials=1000,random_state=9999)
  elif model == 'Lasso':
    from sklearn.linear_model import Lasso
    lin = Lasso(alpha=0.0001,precompute=True,max_iter=1000,
                positive=True, random_state=9999, selection='random')

  lin.fit(reference, signal)
  reference_fitted = lin.predict(reference)
  reference_fitted = reference_fitted.reshape(len(reference_fitted),)

  a = np.empty((i0+1,))
  a[:] = np.nan
  reference_fitted = np.r_[a,reference_fitted]

  return reference_fitted

def calculate_dff(signal,reference,standardized=True):

  if standardized:
    dFF = signal - reference
  else:
    dFF = (signal - reference) / reference

  dFF = standardize_signal(dFF)

  return dFF

def create_perievents(signal,time_,event,window=[-5.0,5.0],dur=None,iei=None,avg_win=None):

  period = find_avg_period(time_, time_format='total seconds')

 # Remove events at the beginning and end of test that less than win
  event = event[np.all(event > abs(window[0]), axis=1)]
  event = event[np.all(event < max(time_)-window[1], axis=1)]

 # Events with one occurence ---------------------------------------------------
  if event.shape[1]==1:
    Array = []
    for e in event:
      s_event = chunk_signal(signal,e,time_,window)
      if avg_win is not None:
        s_event_mean = (s_event[int((-window[0]+avg_win[0])/period):int((-window[0]+avg_win[1])/period)]).mean()
        s_event = s_event - s_event_mean
      Array.append(s_event)
    Array = np.array(Array).squeeze()
    if Array.ndim == 1:
      Array = Array.reshape(1,len(Array))
    Array = Array[~np.isnan(Array).any(axis=1)]

    Perievents = {'onset': Array}

 # Events with onset and offset ------------------------------------------------
  elif event.shape[1]==2:

 # Remove short intervals and durations
    event = adjust_intervals_durations(event, iei, dur)

   # Create Perievent Arrays
   # Initialize Arrays
    Array_onset = []
    Array_offset = []
   # Loop through all onsets and offsets
    for e0,e1 in event:
      s_onset = chunk_signal(signal,e0,time_,window)
      s_offset = chunk_signal(signal,e1,time_,window)
     # Normalize signals to signals in avg_win
      if avg_win is not None:
        s_event_mean = (s_onset[int((-window[0]+avg_win[0])/period):int((-window[0]+avg_win[1])/period)]).mean()
        s_onset = s_onset - s_event_mean
        s_offset = s_offset - s_event_mean
     # Append to arrays
      Array_onset.append(s_onset)
      Array_offset.append(s_offset)
   # Squeeze to 2D arrays
    Array_onset = np.array(Array_onset).squeeze()
    Array_offset = np.array(Array_offset).squeeze()
   # Reshape Arrays if squeezed to 1D
    if Array_onset.ndim == 1:
      Array_onset = Array_onset.reshape(1,len(Array_onset))
      Array_offset = Array_offset.reshape(1,len(Array_offset))
   # Remove elements with nans
    Array_onset = Array_onset[~np.isnan(Array_onset).any(axis=1)]
    Array_offset = Array_offset[~np.isnan(Array_offset).any(axis=1)]

    Perievents = {'onset': Array_onset,
                 'offset': Array_offset}

  return Perievents

def chunk_signal(signal, t0, t, w):

  idx = find_idx(t0, t, 'total seconds')

  period = find_avg_period(t, 'total seconds')

  i0 = idx + int(w[0]/period - 1/2)
  i1 = idx + int(w[1]/period + 1/2) + 1

  chunk = signal[i0:i1]

  return chunk

def create_centered_time_vector(period=0.1,window=[-5.0,5.0]):

  t_pre = np.arange(-period,window[0]-period/2,-period)
  t_post = np.arange(0,window[1]+period/2,period)
  t = np.concatenate([t_pre[-1::-1],t_post])

  return t

def calculate_auc(means, period=0.10, window=[-5.0,5.0], time_frames=[[-3,0],[0,3]]):

  from sklearn import metrics

  t = create_centered_time_vector(period,window)

  stats = np.zeros([len(means),len(time_frames)])

  for i,frame in enumerate(time_frames):
    idx = [i for i in range(len(t)) if t[i]>frame[0] and t[i]<frame[1]]
    #stats1 = np.mean(means[:,idx],axis=1)
    for m in range(len(means)):
      auc = metrics.auc(t[idx],means[m,idx])
      stats[m,i] = auc / (frame[1] - frame[0])

  return stats

max_stars_repo_name: katemartian/FiberPhotometryDataAnalysis

max_issues_repo_name: katemartian/FiberPhotometryDataAnalysis

max_forks_repo_name: katemartian/FiberPhotometryDataAnalysis

reason: import numpy,from scipy


text:
"""Plot variables of various measurement devices over time."""

# Standard library
import datetime
from pprint import pprint

# Third-party
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import matplotlib.units as munits
import numpy as np
import pandas as pd

converter = mdates.ConciseDateConverter()
munits.registry[np.datetime64] = converter
munits.registry[datetime.date] = converter
munits.registry[datetime.datetime] = converter

# First-party
from plot_profile.utils.stations import sdf
from plot_profile.utils.utils import colour_dict
from plot_profile.utils.utils import save_fig
from plot_profile.utils.variables import vdf

# from ipdb import set_trace

def create_plot(
    data,
    multi_axes,
    location,
    start,
    end,
    ymin,
    ymax,
    colours,
    grid,
    show_marker,
    datatypes,
    outpath,
    appendix,
    verbose,
):
    """Create timeseries plot.

    Args:
        data (dict): dictionary, containing one dataframe for each device
        multi_axes (bool): plot 2 yaxis if True
        devices (list): list of devices
        variables (list): list of variables (w/ same unit) to be added to plot
        location (str): abbreviation of station name
        start (datetime obj): start time
        end (datetime obj): end time
        ymin (tuple): y-min values
        ymax (tuple): y-max values
        colours (tuple): User-specified colours
        grid (bool): add grid to plot or not
        show_marker (bool): add marker to model plots or not
        datatypes (tuple): output data types
        outpath (str): output folder path
        appendix (bool): add appendix to output name
        verbose (bool): print details

    """
    # get location dataframe
    loc = sdf[location]
    devices = data.keys()

    # overwrite colour-dict with user-specified colours
    if isinstance(colours, tuple):
        for i, colour in enumerate(colours):
            colour_dict[i] = colour

    # prepare figure
    fig, left_ax = plt.subplots(1, 1, figsize=(8, 5), constrained_layout=True)
    if multi_axes:
        right_ax = left_ax.twinx()
        if verbose:
            print("Creating right axis.")
    if grid:
        # align ticks by implementing ideas from:
        # https://stackoverflow.com/questions/20243683/matplotlib-align-twinx-tick-marks
        left_ax.grid(visible=True)
        # if multi_axes:
        #     right_ax.grid(visible=True)

    # apply limits to the y-axis/axes if some have been specified
    if ymin and ymax:
        if not multi_axes:
            left_ax.set_ylim(ymin[0], ymax[0])
            if len(ymin) != len(ymax):
                print(
                    f"WARNING: inconsistent number of y-axes limits provided! #min: {len(ymin)}, #max: {len(ymax)}"
                )
                print(f"Applied limits: ymin = {ymin[0]}, ymax = {ymax[0]}")

        else:  # there are two y axes
            if (
                len(ymin) == 1 and len(ymax) == 1
            ):  # user provided one ymin/ymax pair --> apply to the left axis
                left_ax.set_ylim(ymin[0], ymax[0])
            elif (
                len(ymin) == 2 and len(ymax) == 2
            ):  # user provided 2 ymin/xmax pairs --> apply to both axis
                left_ax.set_ylim(ymin[0], ymax[0])
                right_ax.set_ylim(ymin[1], ymax[1])
            else:  # ymin/ymax flags have been used iconsistently --> print warning
                print(
                    f"WARNING: inconsistent number of y-axes limits provided! #min: {len(ymin)}, #max: {len(ymax)}"
                )
                print(f"No y-axes limits have been applied.")

    left_ax.set_xlim(start, end)
    title = f"{loc.long_name}: {start.strftime('%d. %b, %H:%M')} - {end.strftime('%d. %b, %H:%M')}"
    left_ax.set_title(label=title)
    left_unit, right_unit = None, None

    # plotting
    colour_index = 0
    actual_devices = []
    for i, device in enumerate(devices):
        model = False
        # 1) retrieve df
        df = data[device]
        if verbose:
            print(i, device)
            pprint(df)

        # x-axis information: dates/timestamps
        dates = pd.to_datetime(df["timestamp"], format="%Y-%m-%d %H:%M:%S")

        # check if there are more than one variable in this dataframe
        if verbose:
            if len(df.columns) > 2:
                print(f"More than one variable in the df for {device}")
            else:
                print(f"Only one variable in the df for {device}")

        # iterate over the c
        for (variable, columnData) in df.iteritems():
            if variable == "timestamp":
                continue

            if verbose:
                print(f"  Variable: {variable}")

            # extract current variable
            if "icon" in device:
                # i.e. 2m_temp could be an icon variable w/o '~'; so make sure the model variable has the correct value!
                model = True
                level = None

            # specify marker
            if model and show_marker:
                marker = "d"
            else:
                marker = None

            # it is only possible for ICON variables to have '~' in them, because a level has to be specified.
            if "~" in variable:
                var, level = (variable.split(sep="~"))[0], (variable.split(sep="~"))[1]
                variable = var
                model = True

            variable = vdf[variable]
            unit = variable.unit
            var_long = variable.long_name
            y = columnData.values

            if model:
                if device.split("~")[1] != "0":
                    if not level:
                        label = f"{var_long}: {device.split('~')[0].upper()} {device.split('~')[1].upper()}"
                    else:
                        label = f"{var_long}: {device.split('~')[0].upper()} {device.split('~')[1].upper()} (Level: {level})"
                else:
                    if not level:
                        label = f"{var_long}: {device.split('~')[0].upper()}"
                    else:
                        label = f"{var_long}: {device.split('~')[0].upper()} (Level: {level})"

            # for observations, the label looks a bit differently
            if not model:  # 'icon' not in device and 'arome' not in device:
                device = device.split("~")[0]
                if "_" in device:
                    label = f"{var_long}: OBS @ {device.split('_')[0].upper()} {device.split('_')[1].upper()}"
                else:
                    label = f"{var_long}: OBS @ {device.upper()}"

            # define unit for the left axes
            if not left_unit:
                left_unit = unit
                left_ax.set_ylabel(f"{left_unit}")

            # define unit for the right axes
            if (not right_unit) and (unit is not left_unit):
                right_unit = unit
                right_ax.set_ylabel(f"{right_unit}")

            # choose correct axes for the current variable and plot data
            if unit == left_unit:
                left_ax.plot(
                    dates,
                    y,
                    color=colour_dict[colour_index],
                    linestyle="-",
                    marker=marker,
                    label=label,
                )
            if unit == right_unit:
                right_ax.plot(
                    dates,
                    y,
                    color=colour_dict[colour_index],
                    linestyle="-",
                    marker=marker,
                    label=label,
                )
            colour_index += 1

    # add legends
    if multi_axes:
        h1, l1 = left_ax.get_legend_handles_labels()
        h2, l2 = right_ax.get_legend_handles_labels()
        left_ax.legend(h1 + h2, l1 + l2, fontsize="small")
    else:
        left_ax.legend(fontsize="small")

    # filename
    start_str = start.strftime("%y%m%d_%H")
    end_str = end.strftime("%y%m%d_%H")

    var_dev = ""
    for key, df in data.items():

        if "icon" in key:
            # a) keys: "icon~ref", "icon~0", "2m~cbh", "2m_tower~temp"
            # remove "0" for model-levels
            if "~0" in key:
                key = key.split(sep="~")[0]
            var_dev += f"_{key}"

            # b) columns: "clct", "sw_up", "temp"
            columns = df.columns
            for column in columns:
                if column != "timestamp":
                    var_dev += f"_{column}"

        # elif 'arome' in key:

        else:  # now its actually a device --> remove variable from key
            var_dev += f"_{key.split('~')[0]}_{key.split('~')[1]}"

        # # b) columns: "clct", "sw_up", "temp"
        # columns = df.columns
        # for column in columns:
        #     if column != "timestamp":
        #         var_dev += f"_{column}"

    filename = f"timeseries_{start_str}-{end_str}_{loc.short_name}{var_dev}"
    save_fig(filename, datatypes, outpath, fig=fig)
    plt.clf()
    return


text:
import sys
import io
import base64
from PIL import Image, ImageOps
import json
import numpy as np

from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms

from .util_image import get_square, hwc_to_chw, normalize
from .util_labelme import shapes_to_label, img_b64_to_arr
from Utils.pillowhelper import rowcolumn2coor

shape_image = (2048, 2048)

def get_image_array_from_json(dir_json, num_channels):
    with open(dir_json, "r") as f:
        data = json.load(f)
    imageData = data['imageData']
    img = img_b64_to_arr(imageData)
    if num_channels == 1:
        img = np.expand_dims(np.dot(img[...,:3], [0.2989, 0.5870, 0.1140]), axis=2)
    assert img.shape[:2] == shape_image
    return img

def get_pil_img_and_mask_from_json(dir_json, label_name_to_value):
    with open(dir_json, "r") as f:
        data = json.load(f)
    img_b64 = data['imageData']
    f = io.BytesIO()
    f.write(base64.b64decode(img_b64))
    img_pil = Image.open(f)

    mask_shapes = data['shapes']
    mask = shapes_to_label(img_pil.size, mask_shapes, label_name_to_value)
    return img_pil, mask

def get_image_mask_from_json(dir_json, label_name_to_value):
    """ Parse a json file to get both image and its corresponding mask (groundtruth).
    Notes:
    (1) In the json file, images have three channels and shape of image array is like (H,W,C);
    (2) Number of classes is limited to 256 since we are using uint8
    (3) The data type of mask should be float since it will be compared with prediction whose
        data type is float
    Arg:
    ------

    Return:
    -------
    img_array, mask
    """
    with open(dir_json, "r") as f:
        data = json.load(f)
    imageData = data['imageData']
    img_array = img_b64_to_arr(imageData)
    image_size = img_array.shape
    mask = shapes_to_label(image_size, data['shapes'], label_name_to_value)
    return img_array, mask

def get_mask_array_from_json(dir_json, label_name_to_value):
    with open(dir_json, "r") as f:
        data = json.load(f)
    mask = shapes_to_label(shape_image, data['shapes'], label_name_to_value)
    return mask

def cropped_imgs_to_patch(ids, num_channels):
    for id, pos in ids:
        im = get_image_array_from_json(id, num_channels)
        yield get_square(im, pos)

def cropped_mask_to_patch(ids, label_name_to_value):
    for id, pos in ids:
        im = get_mask_array_from_json(id, label_name_to_value)
        yield get_square(im, pos)

def dataset_generator(ids, label_name_to_value, num_channels):
    imgs = cropped_imgs_to_patch(ids, num_channels)
    imgs = map(hwc_to_chw, imgs)
    imgs = map(normalize, imgs)
    masks = cropped_mask_to_patch(ids, label_name_to_value)
    return zip(imgs, masks)

class DatasetDefocus(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        image_patch_tensor = transforms.Compose([
            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225]),
            ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample

class DatasetComet(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        # image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        image_patch_tensor = transforms.Compose([
            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225]),
            ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample

class DatasetSmudge(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        # image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        image_patch_tensor = transforms.Compose([
            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225]),
            ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample

class DatasetCactus(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        # image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        image_patch_tensor = transforms.Compose([
            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225]),
            ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample

class DatasetStreak(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        # image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        image_patch_tensor = transforms.Compose([
            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225]),
            ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample

class DatasetEdgeArcing(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        # image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        image_patch_tensor = transforms.Compose([
            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225]),
            ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample

class DatasetMaskLoss(Dataset):
    def __init__(self, filename_with_patch, image_size, patch_size, label_name_to_value,
                 flag_color_jitter=False, flag_equalizer=False):
        self.fn_with_patch = filename_with_patch
        self.image_size = image_size
        self.patch_size = patch_size
        self.label_name_to_value = label_name_to_value
        self.flag_color_jitter = flag_color_jitter
        self.flag_equalizer = flag_equalizer

    def __len__(self):
        return len(self.fn_with_patch)

    def __getitem__(self, idx):
        fn_patch = self.fn_with_patch[idx]
        fn, patch_idx = fn_patch[0], fn_patch[1]
        image, mask = get_pil_img_and_mask_from_json(fn, self.label_name_to_value)
        if self.flag_equalizer:
            image = ImageOps.equalize(image)
        coor_pil = rowcolumn2coor(patch_idx[0], patch_idx[1], self.patch_size)
        image_patch = image.crop(coor_pil)
        mask_patch = np.array(Image.fromarray(mask).crop(coor_pil)).astype(np.float32)
        if self.flag_color_jitter:
            image_patch_tensor = transforms.Compose([
                transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
                transforms.ToTensor(),
                transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225]),
                ])(image_patch)
        else:
            image_patch_tensor = transforms.Compose([
                transforms.ToTensor(),
                transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225]),
                ])(image_patch)

        sample = {'image': image_patch_tensor, 'mask': mask_patch, 'filename': fn, 'rowcol': patch_idx}
        return sample


text:
"""
Author: Salva Rühling Cachay
"""

import os
import torch
import numpy as np
from sklearn.metrics import mean_squared_error
# from utilities.plotting import plot_time_series

def reload_all(model_dir, ens_dict, checkpoint_IDs=None, device='cuda', data_dir='./Data/'):
    from graphino.GCN.GCN_model import GCN
    from graphino.training import evaluate, get_dataloaders
    if checkpoint_IDs is None:
        checkpoint_IDs = ['last']

    try:
        model_dict = torch.load(os.path.join(model_dir, 'last_model.pkl'))
    except FileNotFoundError:
        try:
            model_dict = torch.load(os.path.join(model_dir, '50ep_model.pkl'))
        except FileNotFoundError as e:
            print(e)
            return None, ens_dict

    params, net_params = model_dict['metadata']['params'], model_dict['metadata']['net_params']
    params['data_dir'] = data_dir
    print(net_params)

    (adj, static_feats, _), (_, valloader, testloader) = get_dataloaders(params, net_params)
    model = GCN(net_params, static_feat=static_feats, adj=adj, verbose=False)
    Y = None
    for ckpt in checkpoint_IDs:
        try:
            model_dict = torch.load(os.path.join(model_dir, f'{ckpt}_model.pkl'))
        except FileNotFoundError as e:
            print(e)
            continue
        try:
            model.load_state_dict(model_dict['model'])
        except RuntimeError as e:
            print(e)
            continue

        model.eval()
        model.to(device)

        _, stats = evaluate(valloader, model, device, return_preds=False)
        _, test_stats, Y, preds = evaluate(testloader, model, device, return_preds=True)
        print(model_dir.split('/')[-1], ckpt, "Val. rmse=", stats['rmse'],
              'TEST:', test_stats['rmse'], "Corrcoef, all-season-corrcoef =", test_stats['corrcoef'], test_stats['all_season_cc'])
        if stats['rmse'] > ens_dict[ckpt][-1][0]:
            print("Skip this one")
            continue
        ens_dict[ckpt][-1] = (stats['rmse'], preds)
        ens_dict[ckpt] = sorted(ens_dict[ckpt], key=lambda tup: tup[0])

    return Y, ens_dict

def ensemble_performance(out_dir, verbose=True, device="cuda", num_members=4, checkpoint_IDs=None, ID=None, data_dir='./Data/'):
    """
    :param device:
    :param num_members:
    :return: Tuple of
           i) Member predictions in descending order of preditive skill (wrt val set), i.e. the first has highest skill
           ii) Ground truth
    """
    if not verbose:
        print("This may take a while...")
    if checkpoint_IDs is None:
        checkpoint_IDs = ['last']
    topK = {name: [(10000, None) for _ in range(num_members)]
            for name in checkpoint_IDs
            }
    config_dir = out_dir
    added = False
    for fileID in os.listdir(config_dir):
        if ID is not None and ID not in fileID:
            print('Skipping this one')
            continue
        mdl_dir = os.path.join(config_dir, fileID)
        Y, topK = reload_all(mdl_dir, topK, device=device, checkpoint_IDs=checkpoint_IDs, data_dir=data_dir)
        if Y is not None:
            Ytrue = Y
        added = True

    assert added
    for name in checkpoint_IDs:
        member_preds = [mem[1] for mem in topK[name]]
        stats = ensemble(Ytrue, *member_preds)
        rmse_ens, cc_ens, cc2a = stats['rmse'], stats['corrcoef'], stats['all_season_cc']
        if verbose:
            print(f"ENSEMBLE PERFORMANCE {name} -- RMSE: {rmse_ens:.5f}, Corrcoef = {cc_ens:.5f}, All-season-CC={cc2a:.5f}")
    return member_preds, Ytrue

def rmse(y, preds):
    """
    :return:  The root-mean-squarred error (RMSE)  value
    """
    return np.sqrt(mean_squared_error(y, preds))

def evaluate_preds(Ytrue, preds, **kwargs):
    oni_corr = np.corrcoef(Ytrue, preds)[0, 1]
    try:
        rmse_val = rmse(Ytrue, preds)
    except ValueError as e:
        print(e)
        rmse_val = -1
    # r, p = pearsonr(Ytrue, preds)   # same as using np.corrcoef(y, yhat)[0, 1]
    oni_stats = {"corrcoef": oni_corr, "rmse": rmse_val}  # , "Pearson_r": r, "Pearson_p": p}

    try:
        # All season correlation skill = Mean of the corrcoefs for each target season
        # (whereas the corrcoef above computes it directly on the whole timeseries).
        predsTS = preds.reshape((-1, 12))
        YtestTT = Ytrue.reshape((-1, 12))
        all_season_cc = 0
        for target_mon in range(12):
            all_season_cc += np.corrcoef(predsTS[:, target_mon], YtestTT[:, target_mon])[0, 1]
        all_season_cc /= 12
        oni_stats['all_season_cc'] = all_season_cc
    except ValueError:
        pass

    return oni_stats

def ensemble(Ytrue, *args, return_preds=False, **kwargs):
    members = [mem for mem in args if mem is not None]
    n_members = len(members)
    assert n_members > 0, "An ensemble requires at least 1 member"
    preds = np.zeros(Ytrue.shape)
    for member in members:
        preds = preds + member
    preds = preds / n_members
    stats = evaluate_preds(Ytrue, preds, **kwargs, return_dict=True)
    if return_preds:
        return stats, preds
    return stats

# %%

if __name__ == '__main__':
    print("Torch uses CUDA?", torch.cuda.is_available())
    from utilities.utils import set_gpu
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--ID", default='', type=str)
    parser.add_argument("--gpu_id", default=2, type=int)
    parser.add_argument("--horizon", default=6, type=int)  # number of lead months
    parser.add_argument("--type", default='50ep', type=str)
    parser.add_argument("--data_dir", default='./Data/', type=str)
    args = parser.parse_args()
    set_gpu(args.gpu_id)

    out = f'out/{args.horizon}lead/'
    checkpoint = [args.type]
    ens_members, Y = ensemble_performance(
        out, verbose=True, num_members=4, checkpoint_IDs=checkpoint, ID=args.ID, data_dir=args.data_dir
    )

max_stars_repo_name: salvaRC/Graphino

max_issues_repo_name: salvaRC/Graphino

max_forks_repo_name: salvaRC/Graphino


text:
__all__ = ['denoise_image']

from .. import lib
from . import process_args as pargs
from .get_mask import get_mask

def denoise_image(img, mask=None, shrink_factor=1, p=1, r=3, noise_model='Rician'):
    """
    Arguments
    ---------
    noise_model : string
        'Rician' or 'Gaussian'

    Example
    -------
    >>> import ants
    >>> import numpy as np
    >>> img = ants.image_read(ants.get_ants_data('r16'), 'float')
    >>> # add fairly large salt and pepper noise
    >>> imgnoise = img + np.random.randn(*img.shape).astype('float32')*5
    >>> imgdenoise = ants.denoise_image(imgnoise, ants.get_mask(img))
    """
    inpixeltype = img.pixeltype
    outimg = img.clone('float')

    if mask is None:
        mask = get_mask(img)

    mydim = img.dimension
    myargs = {
        'd': mydim,
        'i': img,
        'n': noise_model,
        'x': mask.clone('unsigned char'),
        's': int(shrink_factor),
        'p': p,
        'r': r,
        'o': outimg,
        'v': 0
    }
    processed_args = pargs._int_antsProcessArguments(myargs)
    lib.DenoiseImage(processed_args)
    return outimg.clone(inpixeltype)


text:
#
# Last modified on Tue Sep  4 17:02:59 PDT 2001 by lindy
#
# $Header: /opt/cvs/python/packages/share1.5/mglutil/math/ncoordstest.py,v 1.2.12.1 2016/02/11 19:04:10 annao Exp $
#

"""Unit test for ncoords.py

Requirements for ncoords.py:
    A. __init__:
        1. make Numeric, homogenious coordinates out of refCoords
        2. raise ValueError if refCoords is bad
    B. reset():
        3. nx3 slice of resultCoords must be equal to refCoords
    C. getResultCoords:
        4. return nx3 (not nx4) coordinates
        5. return as numpy.array or ListType accorinding to self.tolist
"""

from mglutil.math.ncoords import Ncoords
import unittest, math
import numpy
import numpy.random as RandomArray

class NcoordsTest(unittest.TestCase):
    def setUp(self):
        """Called for every test."""

        npts = 500
        dim = 3
        self.max = 9999999.
        self.min = -self.max
        self.random_points = RandomArray.uniform(self.min,
                                                 self.max, (npts,dim)).tolist()

    def tearDown(self):
        pass

class InputOutputValues(NcoordsTest):

    def test_constructor_shape(self):
        """__init__         -- make refCoords and resultCoords homogeneous"""
        n = len(self.random_points)
        ncoords = Ncoords( self.random_points) ### tested call ###
        # confirm shape to be nx4
        self.assertEqual( (n, 4), numpy.shape(ncoords.resultCoords))
        self.assertEqual( (n, 4), numpy.shape(ncoords.refCoords))
        # cofirm that the last column is all ones
        self.assertEqual(numpy.ones(n).tolist(),
                         ncoords.resultCoords[:,3].tolist())
        self.assertEqual(numpy.ones(n).tolist(),
                         ncoords.refCoords[:,3].tolist())

    def test_input_error(self):
        """__init__         -- ValueError on bad input"""
        self.assertRaises(ValueError, Ncoords, list(range(10)))
        self.assertRaises(ValueError, Ncoords, [(1,1,1),(1,1)] )

    def test_reset_values(self):
        """reset            -- points equal input values after reset"""
        nc = Ncoords( self.random_points, tolist=1)
        nc.reset() ### tested call ###
        result = nc.getResultCoords()
        # compare input and output point lists
        self.assertEqual( self.random_points, result)

    def test_getResultCoords_shape(self):
        """getResultCoords  -- if tolist: return nx3 ListType"""
        n = len(self.random_points)
        nc = Ncoords(self.random_points, tolist=0)
        nc.tolist=1
        result = nc.getResultCoords() ### tested call ###
        # confirm shape
        self.assertEqual((n, 3), numpy.shape(result))
        # confirm type
        self.assertEqual(type([]), type(result))

    def test_getResultCoords_type(self):
        """getResultCoords  -- if not tolist: return nx4 numpy.array"""
        n = len(self.random_points)
        nc = Ncoords(self.random_points, tolist=1)
        nc.tolist=0
        result = nc.getResultCoords() ### tested call ###
        # confirm shape
        self.assertEqual((n, 4), numpy.shape(result))
        # confirm type
        self.assertEqual(type(numpy.array([])), type(result))

if __name__ == '__main__':
    unittest.main()

# for example:
#     py mglutil/math/ncoordstest.py -v
# or, to redirect output to a file:
#     py ncoordstest.py -v > & ! /tmp/nct.out

max_stars_repo_name: e-mayo/mscreen


text:
import keras.backend as K
import numpy as np
from kfs.layers.convolutional import (Convolution2DEnergy_TemporalBasis,
                                    Convolution2DEnergy_TemporalCorrelation)

def _test_smoke(channel_order=None):

    from kfs.layers.convolutional import Convolution2DEnergy_TemporalBasis
    from keras.models import Sequential
    #from keras.layers import Flatten, Dense
    input_shape = (12, 3, 64, 64)
    if channel_order is None:
        channel_order = K.image_data_format()
    if channel_order == 'channels_last':
        input_shape = (12, 64, 64, 3)

    rng = np.random.RandomState(42)
    datums = rng.randn(6, 12, 3, 64, 64).astype('float32')
    if channel_order == 'channels_last':
        datums = datums.transpose(0, 1, 3, 4, 2)

    nn2 = Sequential()
    nn2.add(Convolution2DEnergy_TemporalCorrelation(8, 16, 4, (5, 5), 7,
                                            padding='same',
                                            temporal_kernel_size=5,
                                            input_shape=input_shape))
    nn2.compile(loss='mse', optimizer='sgd')

    pred2 = nn2.predict(datums)

    return nn2, nn2.predict(datums)

def test_smoke_channels_first():
    K.set_image_data_format('channels_first')
    _test_smoke('channels_first')

def test_smoke_channels_last():
    K.set_image_data_format('channels_last')
    _test_smoke('channels_last')

def _test_equivalence(channel_order=None):

    from kfs.layers.convolutional import Convolution2DEnergy_TemporalBasis
    from keras.models import Sequential
    #from keras.layers import Flatten, Dense
    input_shape = (12, 3, 64, 64)
    if channel_order is None:
        channel_order = K.image_data_format()
    if channel_order == 'channels_last':
        input_shape = (12, 64, 64, 3)

    nn = Sequential()
    nn.add(Convolution2DEnergy_TemporalBasis(8, 16, 4, (5, 5), 7,
                                            padding='same',
                                            input_shape=input_shape,
                                            data_format=channel_order))

    rng = np.random.RandomState(42)
    datums = rng.randn(6, 12, 3, 64, 64).astype('float32')
    if channel_order == 'channels_last':
        datums = datums.transpose(0, 1, 3, 4, 2)

    nn.compile(loss='mse', optimizer='sgd')

    nn2 = Sequential()
    nn2.add(Convolution2DEnergy_TemporalCorrelation(8, 16, 4, (5, 5), 7,
                                            padding='same',
                                            input_shape=input_shape,
                                            data_format=channel_order))
    nn2.compile(loss='mse', optimizer='sgd')
    nn2.set_weights(nn.get_weights())

    pred1 = nn.predict(datums)
    pred2 = nn2.predict(datums)
    assert ((pred1 - pred2) == 0.).all()

    return nn, nn.predict(datums), nn2, nn2.predict(datums)

def test_equivalence_channels_first():
    K.set_image_data_format('channels_first')
    _test_equivalence('channels_first')

def test_equivalence_channels_last():
    K.set_image_data_format('channels_last')
    _test_equivalence('channels_last')

max_stars_repo_name: the-moliver/kfs

max_issues_repo_name: the-moliver/kfs

max_forks_repo_name: the-moliver/kfs


text:
# pydfs_lineup_optimizer_extras/tests/test_extras.py
# -*- coding: utf-8 -*-
# Copyright (C) 2021 Eric Truett
# Licensed under the MIT License

import numpy as np
from numpy.core.numeric import ones
import pandas as pd

import pytest

from pydfs_lineup_optimizer_extras import *

max_stars_repo_name: sansbacon/pydfs-lineup-optimizer-extras

max_issues_repo_name: sansbacon/pydfs-lineup-optimizer-extras

max_forks_repo_name: sansbacon/pydfs-lineup-optimizer-extras

reason: import numpy,from numpy