text:
# # # # # # # # # # # # # # # # # # # # # '''
# # # # # # # # # # # # # # # # # # # # # Values	Count_Range
# # # # # # # # # # # # # # # # # # # # # 455	1
# # # # # # # # # # # # # # # # # # # # # 475	2
# # # # # # # # # # # # # # # # # # # # # 479	3
# # # # # # # # # # # # # # # # # # # # # 471	4
# # # # # # # # # # # # # # # # # # # # # 255	1
# # # # # # # # # # # # # # # # # # # # # 220	2
# # # # # # # # # # # # # # # # # # # # # 470	5
# # # # # # # # # # # # # # # # # # # # # 473	6
# # # # # # # # # # # # # # # # # # # # # 151	1
# # # # # # # # # # # # # # # # # # # # # 101	2
# # # # # # # # # # # # # # # # # # # # # 293	3
# # # # # # # # # # # # # # # # # # # # # '''

# # # # # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # # # # df = pd.read_clipboard('\t')

# # # # # # # # # # # # # # # # # # # # # df['bins'] = df['Values'] // 100

# # # # # # # # # # # # # # # # # # # # # df['prob'] = df.iloc[:,1].rolling(window=3).mean()

# # # # # # # # # # # # # # # # # # # # # print(df)

# # # # # # # # # # # # # # # # # # # # # from urllib.request import Request, urlopen
# # # # # # # # # # # # # # # # # # # # # from bs4 import BeautifulSoup
# # # # # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # # # # url = "https://www.countries-ofthe-world.com/world-currencies.html"

# # # # # # # # # # # # # # # # # # # # # req = Request(url, headers={"User-Agent":"Mozilla/5.0"})

# # # # # # # # # # # # # # # # # # # # # webpage=urlopen(req).read()

# # # # # # # # # # # # # # # # # # # # # soup = BeautifulSoup(webpage, "html.parser")

# # # # # # # # # # # # # # # # # # # # # table = soup.find("table", {"class":"codes"})

# # # # # # # # # # # # # # # # # # # # # rows = table.find_all('tr')

# # # # # # # # # # # # # # # # # # # # # columns = [v.text for v in rows[0].find_all('th')]

# # # # # # # # # # # # # # # # # # # # # print(columns) # ['Country or territory', 'Currency', 'ISO-4217']
# # # # # # # # # # # # # # # # # # # # # print

# # # # # # # # # # # # # # # # # # # # # from networkx import convert_matrix as nx

# # # # # # # # # # # # # # # # # # # # # df = nx.to_pandas_edgelist

# # # # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # # # # df = pd.DataFrame({'STREAM':['EAGLE','HAWK','NORTH','HAWK','EAGLE','HAWK','NORTH'],'MAT':['A','D','F','D','C','C','E'],'KIS':['B','D','E','D','A','C','D']})

# # # # # # # # # # # # # # # # # # # # # columns = ["A","B","C","D","E", "F"]

# # # # # # # # # # # # # # # # # # # # # print(df)

# # # # # # # # # # # # # # # # # # # # # a = (pd.crosstab(df.STREAM,[df.MAT, df.KIS], margins=True, margins_name='TOTAL').iloc[:,:-1].reindex(columns, axis=1, fill_value=0).rename_axis(None))
# # # # # # # # # # # # # # # # # # # # # # saved = a.to_csv(index=False)
# # # # # # # # # # # # # # # # # # # # # a['TOT'] = a.sum(axis=1)
# # # # # # # # # # # # # # # # # # # # # a['MEAN'] = a.mean(axis=1).round(2)
# # # # # # # # # # # # # # # # # # # # # def x(i):
# # # # # # # # # # # # # # # # # # # # #     if i >5:
# # # # # # # # # # # # # # # # # # # # #         grade='A'
# # # # # # # # # # # # # # # # # # # # #     else:
# # # # # # # # # # # # # # # # # # # # #         grade='E'
# # # # # # # # # # # # # # # # # # # # #     return grade
# # # # # # # # # # # # # # # # # # # # # a['GRD'] = a.MEAN.apply(x)
# # # # # # # # # # # # # # # # # # # # # print(a)

# # # # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # # # # created sample data - same example row inserted 5 times. Not ideal but just was trying to test
# # # # # # # # # # # # # # # # # # # # # df = pd.DataFrame({"text": [b"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # b"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # b"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # b"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # b"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management"]})

# # # # # # # # # # # # # # # # # # # # # df = pd.DataFrame({"text": [r"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # r"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # r"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # r"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management",
# # # # # # # # # # # # # # # # # # # # # r"COMPETENCIES\nBenefits Administration \xe2\x80\x93 Customer Service \xe2\x80\x93 Cost Control \xe2\x80\x93 Recruiting \xe2\x80\x93 Acquisition Management"]})

# # # # # # # # # # # # # # # # # # # # # path=r'/home/anshul/youtube/stackoverflow/resume_dataset.csv'
# # # # # # # # # # # # # # # # # # # # # path2=r'/home/anshul/youtube/stackoverflow/resume_dataset_cleaned.csv'

# # # # # # # # # # # # # # # # # # # # # # path stores the location of the data file downloaded from kaggle
# # # # # # # # # # # # # # # # # # # # # df = pd.read_csv(path)

# # # # # # # # # # # # # # # # # # # # # # remove the binary 'b' prefix reinstated even though the data is
# # # # # # # # # # # # # # # # # # # # # # read as string during df creation
# # # # # # # # # # # # # # # # # # # # # df['Resume'] = [val[1:].encode('utf-8') for val in df['Resume']]

# # # # # # # # # # # # # # # # # # # # # # create a separate column with multiple decode and encode steps to
# # # # # # # # # # # # # # # # # # # # # # retrieve the final clean version
# # # # # # # # # # # # # # # # # # # # # df['text_clean'] = df['Resume'].apply(lambda x: x.decode('unicode_escape').\
# # # # # # # # # # # # # # # # # # # # #     encode('ascii', 'ignore').decode('utf-8').strip())

# # # # # # # # # # # # # # # # # # # # # df['text_clean'].to_csv(path2)

# # # # # # # # # # # # # # # # # # # # # print(df['text_clean'])

# # # # # # # # # # # # # # # # # # # # # .encode('utf-8').decode('unicode_escape').encode('ascii', 'ignore').strip()

# # # # # # # # # # # # # # # # # # # # # print(df)
# # # # # # # # # # # # # # # # # # # # # print(df.info())

# # # # # # # # # # # # # # # # # # # # # df['text_clean'] = df['Resume'].apply(lambda x: x.decode('unicode_escape').\
# # # # # # # # # # # # # # # # # # # # #                                           encode('ascii', 'ignore').\
# # # # # # # # # # # # # # # # # # # # #                                           strip())
# # # # # # # # # # # # # # # # # # # # # df['text_clean'] = [val.decode('utf-8') for val in df['text_clean']]

# # # # # # # # # # # # # # # # # # # # # with pd.option_context('display.max_rows', 5, 'display.max_columns', 1):  # more options can be specified also
# # # # # # # # # # # # # # # # # # # # # print(df['text_clean'])
# # # # # # # # # # # # # # # # # # # # # print(df['text_clean'])

# # # # # # # # # # # # # # # # # # # # # import pandas as pd
# # # # # # # # # # # # # # # # # # # # # import numpy as np

# # # # # # # # # # # # # # # # # # # # # df = pd.DataFrame(np.arange(12).reshape((4, 3)),
# # # # # # # # # # # # # # # # # # # # #                  index=["a", "b", "c", "d"],
# # # # # # # # # # # # # # # # # # # # #                  columns=["a", "b", "c"])

# # # # # # # # # # # # # # # # # # # # # print(df)

# # # # # # # # # # # # # # # # # # # # # df.mask(df.apply(lambda x: x.name == x.index), inplace=True)

# # # # # # # # # # # # # # # # # # # # # print(df)

# # # # # # # # # # # # # # # # # # # # # sentence = input("Give your sentence: ")
# # # # # # # # # # # # # # # # # # # # # words = sentence.split()
# # # # # # # # # # # # # # # # # # # # # print(words)
# # # # # # # # # # # # # # # # # # # # # characters = 0
# # # # # # # # # # # # # # # # # # # # # for word in words:
# # # # # # # # # # # # # # # # # # # # #     characters += len(word)
# # # # # # # # # # # # # # # # # # # # #     average_word_lengt = characters/len(words)

# # # # # # # # # # # # # # # # # # # # # print(characters)
# # # # # # # # # # # # # # # # # # # # # print(average_word_lengt)

# # # # # # # # # # # # # # # # # # # # '''
# # # # # # # # # # # # # # # # # # # # Category     Value
# # # # # # # # # # # # # # # # # # # # A            1
# # # # # # # # # # # # # # # # # # # # C            5
# # # # # # # # # # # # # # # # # # # # '''

# # # # # # # # # # # # # # # # # # # # '''
# # # # # # # # # # # # # # # # # # # # Category     Value
# # # # # # # # # # # # # # # # # # # # A            1
# # # # # # # # # # # # # # # # # # # # B            10
# # # # # # # # # # # # # # # # # # # # '''

# # # # # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # # # # df1 = pd.DataFrame({"Category": ['A', 'C'], "Value": [1, 5]})
# # # # # # # # # # # # # # # # # # # # # df2 = pd.DataFrame({"Category": ['A', 'B'], "Value": [1, 10]})

# # # # # # # # # # # # # # # # # # # # # print(df1)
# # # # # # # # # # # # # # # # # # # # # print(df2)

# # # # # # # # # # # # # # # # # # # # # df3 = df1.merge(df2, how='outer', on=['Category'], suffixes=('_l', '_r'))
# # # # # # # # # # # # # # # # # # # # # df3['Value'] = df3['Value_l'].fillna(0) + df3['Value_r'].fillna(0)
# # # # # # # # # # # # # # # # # # # # # df3 = df3[['Category', 'Value']]

# # # # # # # # # # # # # # # # # # # # # print(df3)

# # # # # # # # # # # # # # # # # # # # import pandas as pd
# # # # # # # # # # # # # # # # # # # # import numpy as np

# # # # # # # # # # # # # # # # # # # # conversion = [["a",5],["b",1],["c",10]]
# # # # # # # # # # # # # # # # # # # # conversion_table = pd.DataFrame(conversion,columns=['Variable','Cost'])

# # # # # # # # # # # # # # # # # # # # data1 = [[1,"3a+b"],[2,"c"],[3,"2c"]]
# # # # # # # # # # # # # # # # # # # # to_solve = pd.DataFrame(data1,columns=['Day','Q1'])

# # # # # # # # # # # # # # # # # # # # desired = [[1,16],[2,10],[3,20]]
# # # # # # # # # # # # # # # # # # # # desired_table=pd.DataFrame(desired,columns=['Day','Q1 solved'])

# # # # # # # # # # # # # # # # # # # # print(conversion_table)
# # # # # # # # # # # # # # # # # # # # print(to_solve)
# # # # # # # # # # # # # # # # # # # # print(desired_table)

# # # # # # # # # # # # # # # # # # # # equn = to_solve['Q1']

# # # # # # # # # # # # # # # # # # # # val=0

# # # # # # # # # # # # # # # # # # # # for eq in equn:
# # # # # # # # # # # # # # # # # # # #     for i in range(len(eq)):
# # # # # # # # # # # # # # # # # # # #         if(eq[i].isdigit()):
# # # # # # # # # # # # # # # # # # # #             val += int(eq[i])
# # # # # # # # # # # # # # # # # # # #         elif(eq[i].isalpha() & i>0 & eq[i-1].isdigit()):
# # # # # # # # # # # # # # # # # # # #             val *= eq[i]
# # # # # # # # # # # # # # # # # # # #         elif(eq[i].isalpha() & i>0):
# # # # # # # # # # # # # # # # # # # #             if
# # # # # # # # # # # # # # # # # # # #     print()

# # # # # # # # # # # # # # # # # # # # # print(equn)

# # # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # # '''
# # # # # # # # # # # # # # # # # # # month_year  Date_of_basket_entry
# # # # # # # # # # # # # # # # # # # 03/2017 01.04.2005
# # # # # # # # # # # # # # # # # # # 02/2019 01.01.1995
# # # # # # # # # # # # # # # # # # # 07/2017 None
# # # # # # # # # # # # # # # # # # # 02/2017 None
# # # # # # # # # # # # # # # # # # # 04/2017 01.01.2020
# # # # # # # # # # # # # # # # # # # '''

# # # # # # # # # # # # # # # # # # # df = pd.read_clipboard()
# # # # # # # # # # # # # # # # # # # print(df.columns)

# # # # # # # # # # # # # # # # # # # # df['month_year'] = pd.to_datetime(df['month_year'], format="%m/%Y")

# # # # # # # # # # # # # # # # # # # # df.loc[df['Date_of_basket_entry'] == 'None']['Date_of_basket_entry'] =
# # # # # # # # # # # # # # # # # # # # # df['Date_of_basket_entry'] = pd.to_datetime(df[df['Date_of_basket_entry'] == 'None'], format="%m.%d.%Y")

# # # # # # # # # # # # # # # # # # # # print(df)
# # # # # # # # # # # # # # # # # # # import numpy as np
# # # # # # # # # # # # # # # # # # # from datetime import datetime
# # # # # # # # # # # # # # # # # # # # df.loc[:,'Date_of_basket_boolean'] = np.where((df.Date_of_basket_entry.isna()) | (df.Date_of_basket_entry < df.month_year) , 0, 1)

# # # # # # # # # # # # # # # # # # # df['month_year'] = pd.to_datetime(df['month_year'], format="%m/%Y")
# # # # # # # # # # # # # # # # # # # print(df.loc[df['Date_of_basket_entry'] == 'None']['Date_of_basket_entry'])
# # # # # # # # # # # # # # # # # # # df.loc[df['Date_of_basket_entry'] == 'None']['Date_of_basket_entry'] = datetime.today().strftime(format="%m.%d.%Y")
# # # # # # # # # # # # # # # # # # # df['Date_of_basket_entry'] = pd.to_datetime(df['Date_of_basket_entry'], format="%m.%d.%Y")
# # # # # # # # # # # # # # # # # # # # df.loc[:,'Date_of_basket_boolean'] = np.where((df.Date_of_basket_entry.notna()) | (df.Date_of_basket_entry < df.month_year) , 0, 1)

# # # # # # # # # # # # # # # # # # # print(df)

# # # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # # lst = ['UserA', 'UserB', 'UserC', 'UserD','UserB', 'UserC']
# # # # # # # # # # # # # # # # # # lst2 = ['X', 'Y', 'Z', 'Y','X','Z']

# # # # # # # # # # # # # # # # # # dm = pd.DataFrame(list(zip(lst, lst2)), columns =['Name', 'val'])
# # # # # # # # # # # # # # # # # # dm['count']= 1
# # # # # # # # # # # # # # # # # # dm.groupby(['Name','val']).count()

# # # # # # # # # # # # # # # # # # # df_piv = pd.crosstab(df['Name'], df['val'])
# # # # # # # # # # # # # # # # # # df_piv = dm.groupby(['Name','val']).count()['count'].unstack('val', fill_value='')

# # # # # # # # # # # # # # # # # # print(dm)
# # # # # # # # # # # # # # # # # # print(df_piv)

# # # # # # # # # # # # # # # # # '''
# # # # # # # # # # # # # # # # # 0        4.0
# # # # # # # # # # # # # # # # # 1        5.0
# # # # # # # # # # # # # # # # # 2        5.0
# # # # # # # # # # # # # # # # # 3        4.0
# # # # # # # # # # # # # # # # # 4        4.0
# # # # # # # # # # # # # # # # # 82624    3.0
# # # # # # # # # # # # # # # # # 82625    4.5
# # # # # # # # # # # # # # # # # 82626    4.0
# # # # # # # # # # # # # # # # # 82627    5.0
# # # # # # # # # # # # # # # # # 82628    4.5
# # # # # # # # # # # # # # # # # '''

# # # # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # # # df = pd.read_clipboard()
# # # # # # # # # # # # # # # # # df.columns = ['no', 'rating']

# # # # # # # # # # # # # # # # # df['rating'] = df['rating'].astype('int')

# # # # # # # # # # # # # # # # # print(df)

# # # # # # # # # # # # # # # # '''
# # # # # # # # # # # # # # # # ID    ID2       DATE       Status
# # # # # # # # # # # # # # # # 23    10     2019-06-11     Sent
# # # # # # # # # # # # # # # # 23    20     2019-06-21     Sent
# # # # # # # # # # # # # # # # 23    30     2019-06-26     Sent
# # # # # # # # # # # # # # # # 23    40     2019-06-27     Sent
# # # # # # # # # # # # # # # # 23    50     2019-12-02     Sent
# # # # # # # # # # # # # # # # '''

# # # # # # # # # # # # # # # # import pandas as pd
# # # # # # # # # # # # # # # # import datetime as dt
# # # # # # # # # # # # # # # # df = pd.read_clipboard()

# # # # # # # # # # # # # # # # df['FMT_DATE'] = pd.to_datetime(df['DATE'], format="%Y-%m-%d")
# # # # # # # # # # # # # # # # df['YEAR']

# # # # # # # # # # # # # # # # print(df)
# # # # # # # # # # # # # # # # print(df.columns)

# # # # # # # # # # # # # # # # # id = df.groupby(['ID'], as_index = False)
# # # # # # # # # # # # # # # # # id_dict = {}

# # # # # # # # # # # # # # # # # all_df = id.get_group(ID)

# # # # # # # # # # # # # # # # # letter_count = 0
# # # # # # # # # # # # # # # # # for index, row in all_df.iterrows():
# # # # # # # # # # # # # # # # #         if ((row['Status'] == 'Sent')):
# # # # # # # # # # # # # # # # #             letter_count = letter_count + 1

# # # # # # # # # # # # # # # # # id_dict.update({ID:letter_count})
# # # # # # # # # # # # # # # # # df['letter_count'] = df['ID'].map(id_dict)

# # # # # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # # # # def filter(df, foo="df['foo']"):
# # # # # # # # # # # # # # #     newdf = df[df['foo']==eval(foo)]
# # # # # # # # # # # # # # #     print(newdf)

# # # # # # # # # # # # # # # data = {
# # # # # # # # # # # # # # #     'foo': ['First value', 'Second value'],
# # # # # # # # # # # # # # #     'bar': ['First value', 'Second value']
# # # # # # # # # # # # # # #     }

# # # # # # # # # # # # # # # df = pd.DataFrame(data, columns = ['foo', 'bar'])

# # # # # # # # # # # # # # # filter(df, "'First value'") #

# # # # # # # # # # # # # # data = {
# # # # # # # # # # # # # #   "process": "1",
# # # # # # # # # # # # # #   "description": "some more details"
# # # # # # # # # # # # # # }

# # # # # # # # # # # # # # print(data['description'])

# # # # # # # # # # # # # # li = []
# # # # # # # # # # # # # # li2 = ['A', 'B', 'C', 'D']

# # # # # # # # # # # # # # # strg=''

# # # # # # # # # # # # # # # li = [map(lambda x,y:x+y, val,strg) for val in li2]
# # # # # # # # # # # # # # # # [lambda x: x+val for val in li2]
# # # # # # # # # # # # # # # print(li)

# # # # # # # # # # # # # # from itertools import accumulate

# # # # # # # # # # # # # # li = list(accumulate(li2, lambda x,y: x+y))
# # # # # # # # # # # # # # # ['A', 'AB', 'ABC', 'ABCD']

# # # # # # # # # # # # # # print(li)

# # # # # # # # # # # # # zeros = [0, 0, 0, 0]
# # # # # # # # # # # # # outp=[]

# # # # # # # # # # # # # for key, val in enumerate(zeros):
# # # # # # # # # # # # #     temp = zeros.copy()
# # # # # # # # # # # # #     temp[key] = 1
# # # # # # # # # # # # #     outp.append(tuple(temp))

# # # # # # # # # # # # # print(outp)

# # # # # # # # # # # # import pandas as pd

# # # # # # # # # # # # '''
# # # # # # # # # # # #     id  colA colB
# # # # # # # # # # # # 0   1   4    1
# # # # # # # # # # # # 1   2   5    2
# # # # # # # # # # # # 2   3   2    4
# # # # # # # # # # # # 3   4   4    2
# # # # # # # # # # # # 4   5   2    4
# # # # # # # # # # # # '''

# # # # # # # # # # # # '''
# # # # # # # # # # # #     id  colA colB colC
# # # # # # # # # # # # 0   1   4    1    0
# # # # # # # # # # # # 1   2   5    2    0
# # # # # # # # # # # # 2   5   2    4    0
# # # # # # # # # # # # '''

# # # # # # # # # # # # df1 = pd.read_clipboard()

# # # # # # # # # # # # df2 = df1.loc[df1['id'].isin(['1','2','5'])]
# # # # # # # # # # # # df2['colC'] = 0

# # # # # # # # # # # # df3 = df1.loc[~df1['id'].isin(list(df2['id']))]

# # # # # # # # # # # # print(df3)
# # # # # # # # # # # # # print(df2)

# # # # # # # # # # # import pandas as pd

# # # # # # # # # # # alphabet_list = ['a','b','c','d','e','f']

# # # # # # # # # # # master_df = pd.DataFrame()

# # # # # # # # # # # for letter in alphabet_list:

# # # # # # # # # # #     player_df = pd.read_html('https://www.nfl.com/players/active/{}'.format(letter))

# # # # # # # # # # #     master_df = master_df.append(player_df)

# # # # # # # # # # # print(master_df)

# # # # # # # # # # import pandas as pd

# # # # # # # # # # '''
# # # # # # # # # #    id   year    month      target1
# # # # # # # # # # 0  324  2019.0  1.0        100.0
# # # # # # # # # # 1  325  2019.0  3.0        100.0
# # # # # # # # # # 2  326  2019.0  10.0       100.0
# # # # # # # # # # 3  327  2019.0  11.0       100.0
# # # # # # # # # # 5  328  2020.0  4.0       100.0
# # # # # # # # # # 6  329  2020.0  12.0       100.0
# # # # # # # # # # 7  330  2020.0  2.0       100.0
# # # # # # # # # # '''

# # # # # # # # # # df = pd.read_clipboard()

# # # # # # # # # # df1 = df.loc[df['year'] <= 2019].join(df.loc[df['month'] <= 10], how='', lsuffix='_l', rsuffix='_r')  #.loc[df['month'] <= 10]

# # # # # # # # # # print(df1)

# # # # # # # # # # # print(df)

# # # # # # # # # import pyap
# # # # # # # # # test_address = """
# # # # # # # # #    4998 Stairstep Lane Toronto ON
# # # # # # # # #     """
# # # # # # # # # addresses = pyap.parse(test_address, country='CA')
# # # # # # # # # for address in addresses:
# # # # # # # # #         # shows found address
# # # # # # # # #         print(address)
# # # # # # # # #         # shows address parts
# # # # # # # # #         print(address.as_dict())

# # # # # # # # # addresses.apply(lambda x: pyap.parse(x['address'], country='CA'),axis=1)

# # # # # # # # '''
# # # # # # # # productNum  productOMS  productPrice
# # # # # # # # 2463448  1002623073  419.95
# # # # # # # # 2463448  sample_text#@$#%#_test  239.95
# # # # # # # # 2463448  1002623073  4239.95
# # # # # # # # 2463448  sample_text#@$#%#_test  4124.95
# # # # # # # # 2463448  1002623073  249.95
# # # # # # # # 2463448  1002623073  454549.95
# # # # # # # # '''

# # # # # # # # import pandas as pd

# # # # # # # # df = pd.read_clipboard()

# # # # # # # # print(df)

# # # # # # # # df1 = df.loc[df['productOMS'].str.contains('[0-9]+')]

# # # # # # # # print(df1)

# # # # # # # import pandas as pd
# # # # # # # pincodes=[800678,800456]
# # # # # # # numbers=[2567890,256757]
# # # # # # # labels=['R','M']
# # # # # # # first=pd.DataFrame({'Number':numbers, 'Pincode':pincodes},index=labels)
# # # # # # # print(first)

# # # # # # import os
# # # # # # import glob
# # # # # # import pandas as pd

# # # # # # # replace "/dir" with the location that contains the files
# # # # # # os.chdir("/home/anshul/youtube/stackoverflow/csv_files/")

# # # # # # extn = 'csv'
# # # # # # filenames = [i for i in glob.glob('*.{}'.format(extn))]

# # # # # # combined_csv = pd.concat([pd.read_csv(f) for f in filenames ])
# # # # # # combined_csv.to_csv( "/home/anshul/youtube/stackoverflow/csv_files/combined_csv.csv", index=False)

# # # # # # # , encoding='utf-8-sig'

# # # # # '''
# # # # # a,b,4
# # # # # b,c,5
# # # # # a,c,3
# # # # # a,a,0
# # # # # b,b,0
# # # # # c,c,0
# # # # # b,a,4
# # # # # c,b,5
# # # # # c,a,3
# # # # # '''

# # # # # import pandas as pd

# # # # # df = pd.read_clipboard()

# # # # # print(df)

# # # # # df1 = df.as_matrix()

# # # # # print(df1)

# # # # '''
# # # # A  B  C
# # # # 1  2  4
# # # # 2  4  6
# # # # 8  10 12
# # # # 1  3  5
# # # # '''

# # # # import pandas as pd

# # # # df = pd.read_clipboard()

# # # # print(df)

# # # lst1 = ['A1 B1 C1', 'A2 B2 D1', 'S1 M1 A3', 'A4 B3 G1','H1 K1 W1']

# # # import pandas as pd

# # # lst = ['A', 'B', 'C', 'D']
# # # print(''.join(lst))

# # # df = pd.DataFrame(columns=lst)

# # # # print(df)

# # # tmp_dict = {}

# # # for strg in lst1:
# # #     strg = strg.split()
# # #     for i in strg:
# # #         if i[0] in ''.join(lst):

# # #     # for val in lst:
# # #     #     if val in strg

# # import pandas as pd

# # lst1 = ['A1 B1 C1', 'A2 B2 D1', 'S1 M1 A3', 'A4 B3 G1','H1 K1 W1']

# # cols = ['A', 'B', 'C', 'D']   ### cols for each string
# # df = pd.DataFrame(columns=cols)

# # ### Populating values
# # for elt in lst1:
# #     new = {}
# #     for sub_elt in elt.split(" "):
# #         if sub_elt[0] in cols:
# #             new[sub_elt[0]] = sub_elt
# #     df = df.append(pd.Series(new), ignore_index=True)

# # print(df)

# import pandas as pd

# lst1 = ['A1 B1 C1', 'A2 B2 D1', 'S1 M1 A3', 'A4 B3 G1','H1 K1 W1']

# df = pd.DataFrame()

# for lst in lst1:
#     df = df.append([sorted(lst.split())])

# print(df)

max_stars_repo_name: Anshul-GH/stackoverflow

max_issues_repo_name: Anshul-GH/stackoverflow

max_forks_repo_name: Anshul-GH/stackoverflow

reason: import numpy,from networkx


text:
#!/usr/bin/env python
# -*- coding: latin-1 -*-
#
#   Copyright 2016-2019 Blaise Frederick
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
from __future__ import print_function, division

import numpy as np
import pylab as plt

from rapidtide.correlate import aliasedcorrelate, aliasedcorrelator

def test_aliasedcorrelate(display=False):
    Fs_hi = 10.0
    Fs_lo = 1.0
    siginfo = [[1.0, 1.36129345], [0.33, 2.0]]
    modamp = 0.01
    inlenhi = 1000
    inlenlo = 100
    offset = 0.5
    width = 2.5
    rangepts = 101
    timerange = np.linspace(0.0, width, num=101) - width / 2.0
    hiaxis = np.linspace(0.0, 2.0 * np.pi * inlenhi / Fs_hi, num=inlenhi, endpoint=False)
    loaxis = np.linspace(0.0, 2.0 * np.pi * inlenlo / Fs_lo, num=inlenlo, endpoint=False)
    sighi = hiaxis * 0.0
    siglo = loaxis * 0.0
    for theinfo in siginfo:
        sighi += theinfo[0] * np.sin(theinfo[1] * hiaxis)
        siglo += theinfo[0] * np.sin(theinfo[1] * loaxis)
    aliasedcorrelate_result = aliasedcorrelate(sighi, Fs_hi, siglo, Fs_lo, timerange, padvalue=width)

    thecorrelator = aliasedcorrelator(sighi, Fs_hi, Fs_lo, timerange, padvalue=width)
    aliasedcorrelate_result2 = thecorrelator.apply(siglo, 0.0)

    if display:
        plt.figure()
        #plt.ylim([-1.0, 3.0])
        plt.plot(hiaxis, sighi, 'k')
        plt.scatter(loaxis, siglo, c='r')
        plt.legend(['sighi', 'siglo'])

        plt.figure()
        plt.plot(timerange, aliasedcorrelate_result, 'k')
        plt.plot(timerange, aliasedcorrelate_result2, 'r')
        print('maximum occurs at offset', timerange[np.argmax(aliasedcorrelate_result)])

        plt.show()

    #assert (fastcorrelate_result == stdcorrelate_result).all
    aethresh = 10
    #np.testing.assert_almost_equal(fastcorrelate_result, stdcorrelate_result, aethresh)

def main():
    test_aliasedcorrelate(display=True)

if __name__ == '__main__':
    main()

ext: py

max_stars_repo_name: azlau/rapidtide

max_issues_repo_name: azlau/rapidtide

max_forks_repo_name: azlau/rapidtide

reason: import numpy


text:
import numpy as np
import random
import os
from PIL import Image
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches

class NeuralNetwork():
    def __init__(self, layers):
        self.parameters, self.velocity, self.training = {}, {}, True
        self.layer_number = len(layers)
        for l in range(1, self.layer_number): # generate weight and bias according to the input layers
            self.parameters["W" + str(l)] = np.random.random((layers[l - 1], layers[l])) - 0.5
            self.parameters["b" + str(l)] = np.random.random(layers[l]) - 0.5

    def forward(self, X): # do forward propagation
        self.cache = {}
        input = X
        for l in range(1, self.layer_number):
            if self.training:
                self.cache["l" + str(l)] = input # save the input as cache for backpropagation
            output = np.dot(input, self.parameters["W" + str(l)]) + self.parameters["b" + str(l)] # input * weight + bias
            if l < self.layer_number - 1:
                input = sigmoid(output)

        return output

    def criterion(self, y_pd, y_gt): # calculate the loss and gradients
        loss = cross_entropy(softmax(y_pd), one_hot(y_gt)) # get cross entropy loss
        back = (softmax(y_pd) - one_hot(y_gt)) / y_gt.shape
        self.gradients = {}
        if self.training: #  only calculate gradient in training mode
            for l in reversed(range(1, self.layer_number)):
                self.gradients["dw" + str(l)] = np.dot(self.cache["l" + str(l)].T, back)
                self.gradients["db" + str(l)] = np.sum(back, axis=0)
                back = sigmoid_delta(self.cache["l" + str(l)]) * np.dot(back, self.parameters["W" + str(l)].T)

        return loss.mean()

    def step(self, learning_rate, momentum=0.9): # optimize the parameters by minus the gradient
        for l in range(1, self.layer_number):
            try:
                self.velocity["Vw" + str(l)] = self.velocity["Vw" + str(l)] * momentum + self.gradients["dw" + str(l)]
                self.velocity["Vb" + str(l)] = self.velocity["Vb" + str(l)] * momentum + self.gradients["db" + str(l)]
            except: # there is no velocity in the first time, so set the gradients as velocity
                self.velocity["Vw" + str(l)] = self.gradients["dw" + str(l)]
                self.velocity["Vb" + str(l)] = self.gradients["db" + str(l)]
            # update the parameters using velocity
            self.parameters["W" + str(l)] = self.parameters["W" + str(l)] - learning_rate * self.velocity["Vw" + str(l)]
            self.parameters["b" + str(l)] = self.parameters["b" + str(l)] - learning_rate * self.velocity["Vb" + str(l)]

    def max(self, output): # convert the output to predicted class
        predict = []
        for o in output:
            p, = np.where(o == o.max())
            predict.append(p[0])

        return predict

    def no_grad(self): # clear the gradients and cache
        self.cache = {}
        self.gradients = {}

    def eval(self): # turn to evaluation mode
        self.training = False

    def train(self): # turn to train mode
        self.training = True

def get_data(batchsize, preprocess=False): # get datas from path then return
    train_dir, test_dir = "./Data/Data_train", "./Data/Data_test"
    classes = ["Carambula", "Lychee", "Pear"]
    train_data, test_data = [], []
    for i, clas in enumerate(classes):
        trainset = [(np.array(Image.open(os.path.join(train_dir, clas, p))), i) for p in os.listdir(os.path.join(train_dir, clas))]
        testset = [(np.array(Image.open(os.path.join(test_dir, clas, p))), i) for p in os.listdir(os.path.join(test_dir, clas))]
        train_data.extend(trainset), test_data.extend(testset)
    if preprocess:
        train_x, train_y = pre_process(train_data, shuffle=True, normalize=True)
        test_x, test_y = pre_process(test_data, shuffle=True, normalize=True)
    else:
        train_x, train_y = pre_process(train_data, shuffle=False, normalize=False)
        test_x, test_y = pre_process(test_data, shuffle=False, normalize=False)

    return train_x, train_y, test_x, test_y

def pre_process(dataset, shuffle=True, normalize=True): # do pre process to data, e.g. shuffle, normalize
    x, y = [], []
    if shuffle:
        random.shuffle(dataset)
    for (a, b) in dataset:
        if normalize:
            a = a / 255 - 0.5
        x.append(a.reshape(-1)), y.append(b)

    return np.array(x), np.array(y)

def dataloader(x, y, batchsize): # load data as a dataloader
    length = x.shape[0]
    loader = []
    if length % batchsize == 0:
        for i in range(int(length/batchsize)):
            loader.append((x[i*batchsize:(i+1)*batchsize], y[i*batchsize:(i+1)*batchsize]))
    else:
        for i in range(int(length/batchsize)):
            loader.append((x[i*batchsize:(i+1)*batchsize], y[i*batchsize:(i+1)*batchsize]))
        loader.append((x[(i+1)*batchsize:], y[(i+1)*batchsize:]))

    return loader

def softmax(y): # softmax function
    exp_y = np.exp(y)
    sum = np.sum(exp_y, axis=1)

    return exp_y / sum[:, np.newaxis]

def one_hot(y): # one hot encoder for y
    out_y = np.zeros((y.shape[0], 3))
    out_y[np.arange(y.shape[0]), y] = 1

    return out_y

def cross_entropy(y_pd, y_gt, epsilon=1e-10): # calculate cross entropy loss
    loss = -y_gt * np.log10(y_pd + epsilon)

    return loss

def sigmoid(x): # sigmoid function

    return 1 / (1 + np.exp(-x))

def sigmoid_delta(x): # derivative of relu function
    f = sigmoid(x)

    return f * (1 - f)

def plot_decision_region(x, y, filename):
    plt.figure(figsize=(10,6))
    plt.grid(True)
    for input, target in zip(x, y):
        if target == 0:
            plt.scatter(input[0], input[1], c='r', marker='o', s=5)
        elif target == 1:
            plt.scatter(input[0], input[1], c='g', marker='o', s=5)
        elif target == 2:
            plt.scatter(input[0], input[1], c='b', marker='o', s=5)
    red_patch = mpatches.Patch(color='red', label="Carambula")
    green_patch = mpatches.Patch(color='green', label="Lychee")
    blue_patch = mpatches.Patch(color='blue', label="Pear")
    plt.legend(handles=[red_patch, green_patch, blue_patch])
    plt.savefig(filename + ".png")

def plot_loss_acc(loss, acc, e, name, part):
    fig, axs = plt.subplots(1, 2)
    axs[0].plot(e, loss)
    axs[0].set_title(name + " Loss")
    axs[0].set_xlabel("epoch")
    axs[1].plot(e, acc)
    axs[1].set_title(name + " Accuracy")
    axs[1].set_xlabel("epoch")
    axs[0].grid(), axs[1].grid()
    fig.savefig(name + "_" + part + ".png")


text:
from typing import Tuple, List
import numpy as np
import random

def sample_data(X: List, y: np.ndarray) -> Tuple[List, np.ndarray]:
    pos_index = []
    neg_index = []

    for i in range(y.shape[0]):
        if y[i]:
            pos_index.append(i)
        else:
            neg_index.append(i)

    print(
        f"Num positive examples: {len(pos_index)}. Num negative examples: {len(neg_index)}"
    )

    sample_amount = min(len(pos_index), len(neg_index))

    random.seed(8)
    final_index = random.sample(pos_index, sample_amount) + random.sample(
        neg_index, sample_amount
    )
    random.shuffle(final_index)

    return [X[i] for i in final_index], y[final_index]


text:
#!/home/wli/env python3
# -*- coding: utf-8 -*-
"""
Title: patches near tumor
=========================

Created: 10-31-2019
Python-Version: 3.5, 3.6

Description:
------------
This is a library module used for prediction of the whole tissue area
using slide window method with one or several pixel shift each prediction.
The basic idea is to read a bigger tissue region into memory to avoid side
effect of predicting a small 224 x 224 patch.

This module is also used by hard negative mining (main_pred_hnm_workstation
, main_pred_hnm)

"""

from tqdm import tqdm
import numpy as np
import pandas as pd
import os.path as osp
import openslide
import glob
import os
import sys
import skimage.io as io
import re
import staintools
#############################################
import stainNorm_Reinhard
import stainNorm_Macenko

def list_file_in_dir(path, file_ext):
    """
    The function is used to return a list of files in a specific directory and its subdirectories.

    :param str path: the interested directory
    :param str file_ext: file extension. for exaple, 'tif', 'jpg'

    :return: a list of files with full paths
    :rtype: list
    """
    files = [file for file in glob.glob(
        path + "**/*.%s" % file_ext, recursive=True)]
    files.sort()

    return files

def predict_batch_from_model(patches, model):
    """
    There are two values for each prediction: one is for the score of normal patches.
    ; the other one is for the score of tumor patches. The function is used to select
    the score of tumor patches
    :param array patches: a list of image patches to be predicted.
    :param object model: the trained neural network.
    :return lsit predictions: a list of scores for each predicted image patch.
                              The score here is the probability of the image as a tumor
                              image.

    """
    predictions = model.predict(patches)
    # print(predictions[:, 1])
    # print(predictions[:, 0])
    predictions = predictions[:, 1]
    return predictions

def modify_task_id(task_id, slide_category):
    """
    The task number of 150,000 is not enough. This function creates another task_id
    to make prediction for all the slides.
    :param int origin_taskid: the task id coming from HPC
    :param string slide_category: one option from [tumor, normal, test].
    :return new taskid: an ajusted task id
            patches_per_task: the number of patches need to be assigned to a task on HPC
            PatchNumber: a list to show the patch numbber for each slide.
    """

    if slide_category == 'tumor':
        patches_per_task = 180
        task_id = (task_id-1)*180
    elif slide_category == 'normal':
        patches_per_task = 200
        task_id = (task_id-1)*200
    elif slide_category == 'test':
        patches_per_task = 160
        task_id = (task_id-1)*160

    return (task_id, patches_per_task)

#####################################################################################################
# The following function is used to get the slide index and patch id inside a specific slide.
#####################################################################################################

def slide_patch_index(task_id, patches_per_task, PatchNumber):
    """
    This is a critical function to identify which slide and which part of slide will be predicted by
    a specific task id from HPC
    :param int task_id: the mordified task id
    :param int patches_per_task: the number of patches to be predicted per task id
    :return int i: the index for slide that will be predicted
            int j: the index for the part of slide to be predicted
    """

    SlideRange = PatchNumber.ix[(
        PatchNumber['TaskIDrange']-task_id).abs().argsort()[:2]]
    SlideIndex = SlideRange.index.tolist()

    # i is for slide index; j is for patch id
    if task_id < SlideRange.at[SlideIndex[0], 'TaskIDrange']:

        i = SlideIndex[0]

    else:

        i = SlideIndex[0] + 1

    # find the j
    if i == 0:
        j = task_id
        j_dif = 0  # j_dif will not be used if i=0
    else:

        TaskRange_previous = PatchNumber.at[PatchNumber.index[i-1], 'TaskIDrange']
        j_dif = task_id - TaskRange_previous - 1
    # important: the first several patches will not be included if only use j = j_dif

        if j_dif < patches_per_task:

            j = 0

        else:
            j = j_dif

    return (i, j, j_dif)

# what happen if a 160 range cover two slides????????
# find the WSI slide information based on i

def slide_level_param(i, index_path_pred, slide_path_pred):
    """
    The function is used to do slide level operation and return some key parameters and objects.
    :param int i: the index of slide to be predicted
    :param list index_path_pred: the list of all the index file.
    :param list slide_path_pred: the list of all WSIs to be predicted.
    :return dataframe all_samples: all the patches in a slide
            int       n_samples: the number of total patches in a slide
            object    slide: the object from OpenSlide
            string    new_slide_path: the slide path corresponding to index path
    """
    all_samples = pd.read_pickle(index_path_pred[i])

    # find the related WSI slide. new_slide_path is a list.
    new_slide_path = [x for x in slide_path_pred if re.search(
        osp.basename(index_path_pred[i]).replace('.pkl', '.tif'), x)]
    #   new_dim_path  = [x for x in dim_paths if re.search(osp.basename(index_paths[i]).replace('.pkl', '.npy'), x)]
    # new_slide_path = [x for x in slide_paths if re.search(osp.splitext(osp.basename(x))[0], index_path[i])]
    # print(new_slide_path)
    all_samples.slide_path = new_slide_path[0]

    # print(all_samples)
    n_samples = len(all_samples)
    slide = openslide.open_slide(new_slide_path[0])

    return (all_samples, n_samples, slide, new_slide_path[0])

def slide_level_info_hnm(i, index_path_pred, slide_path_pred, ground_truth_paths=False):
    """
    The function is used to do slide level operation and return some key parameters and objects.
    :param int : the index of slide to be predicted
    :param list patch_index_path: the list of all the index files.
    :param list slide_paths: the list of all WSIs to be predicted.
    :param list ground_truth_paths: the list of all ground_truth files that binary file in tif format.
    :return: dataframe all_samples: all the patches in a slide
            int       n_samples: the number of total patches in a slide
            object    slide: the object from OpenSlide
            string    new_slide_path: the slide path corresponding to index path
    """
    all_samples = pd.read_pickle(index_path_pred[i])

    # find the related WSI slide. new_slide_path is a list.
    new_slide_path = [x for x in slide_path_pred if re.search(
        osp.basename(index_path_pred[i]).replace('.pkl', '.tif'), x)]
    #   new_dim_path  = [x for x in dim_paths if re.search(osp.basename(index_paths[]).replace('.pkl', '.npy'), x)]
    # new_slide_path = [x for x in slide_paths if re.search(osp.splitext(osp.basename(x))[0], index_path[])]
    # print(new_slide_path)
    all_samples.slide_path = new_slide_path[0]

    # print(all_samples)
    n_samples = len(all_samples)
    slide = openslide.open_slide(new_slide_path[0])
    # print(n_samples)
    if ground_truth_paths:

        new_ground_truth_path = [x for x in ground_truth_paths if re.search(
            osp.basename(index_path_pred[i]).replace('.pkl', '_mask.tif'), x)]

        ground_truth = openslide.open_slide(new_ground_truth_path[0])

    return (all_samples, n_samples, slide, new_slide_path[0], ground_truth) if ground_truth_paths else (all_samples, n_samples, slide, new_slide_path[0])

####################################################################################################
# create folders to store individual prediction for patches
####################################################################################################

def creat_folder(new_slide_path, path_for_results):
    """
    To create folder to store the prediction results.
    :param string new_slide_path
    :param string path_for_results
    """
    folder_name = osp.splitext(osp.basename(new_slide_path))[0]
    path_to_create = osp.join(path_for_results, folder_name)

    try:
        os.makedirs(path_to_create)
    except:
        print('Folder exists. Skipped')

    return path_to_create

def patches_for_pred(i, j, j_dif, PatchNumber, patches_per_task, all_samples):
    """
    find the subsamples from the pkl table based on task id by using i (slide number) and j (patch number)
    :param int i: the index to find the slide to be predicted.
    :param int j: the index to find the patch series to be predicted in a slide.
    :param int patches_per_task: the number of patches to be predicted in a task on HPC.
    :return object, int subsample: a small portion of the dataframe showing the patch label and path
                        Range_right: the right end of the sample range.
    :note  Range_right is created to avoid the same task will go cross two slides.
    """
    ###
    # step1 : identify the total number of patches for slide i
    ###

    Total_Patches_One_Slide = PatchNumber.at[PatchNumber.index[i], 'PatchNumber']

    #################
    # step2: find the range of patch number inside a slide
    #####################

    #Readin_List = list(range(0, Total_Patches_One_Slide, 150))
    #Readin_List = Readin_List.extend(Total_Patches_One_Slide)

    #patch_Inlist = [x - j for x in Readin_List].argsort()[:2]

    #Patch_Index_left = Readin_List[min(Patch_Index)]

    # for test images
    if j + patches_per_task > Total_Patches_One_Slide:

        Patch_Index_right = Total_Patches_One_Slide

    else:
        # note: if has to be here to make sure the existence of Patch_index_right when i=0
        if i == 0:

            #   for test images
            Patch_Index_right = j + patches_per_task

        else:
            #       for test images
            Patch_Index_right = j_dif + patches_per_task

    # note: the last patch is Patch_Index_right -1 since iloc range does not include the end number.
    SubSamples = all_samples.iloc[j: Patch_Index_right]
    Range_Right = Patch_Index_right - 1

    return (SubSamples, Range_Right)

###################################################################################################
###################################################################################################
###################################################################################################

# all the following functions are called by the last function.
####################################################################################################
# read patches into memory for prediction
####################################################################################################
#g = 1
# create a empty list to store the results from the prediction of 150 patches

# this is read_image function from stain tools, I just copy here and use it directly since sometimes, this function from staintools
# does not work.

# def read_image(path):
#    """
#    Read an image to RGB uint8.
#    Read with opencv (cv) and covert from BGR colorspace to RGB.
#    :param path: The path to the image.
#    :return: RGB uint8 image.
#    """
#    assert os.path.isfile(path), "File not found"
#    im = cv2.imread(path)
#    # Convert from cv2 standard of BGR to our convention of RGB.
#    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
#   return im
# Here we begin to set up a standard image and
# stain normalization is conducted by using vahadane's method.
####
#imagest = staintools.read_image("/home/wli/tumor_st.png")
#imagest = io.imread("/home/wli/Downloads/test/tumor_st.png")
# img = staintools.read_image(tumor_path)
#imagest_standard = standardizer.transform(imagest)
# stain_normalizer.fit(standardizer)

def color_normalization(template_image_path, color_norm_method):
    """
    The function put all the color normalization methods together.
    :param string template_image_path: the path of the image used as a template
    :param string color_norm_method: one of the three methods: vahadane, macenko, reinhard.
    :return object
    """
    template_image = staintools.read_image(template_image_path)
    standardizer = staintools.LuminosityStandardizer.standardize(
        template_image)
    if color_norm_method == 'Reinhard':
        color_normalizer = stainNorm_Reinhard.Normalizer()
        color_normalizer.fit(standardizer)
    elif color_norm_method == 'Macenko':
        color_normalizer = stainNorm_Macenko.Normalizer()
        color_normalizer.fit(standardizer)
    elif color_norm_method == 'Vahadane':
        color_normalizer = staintools.StainNormalizer(method='vahadane')
        color_normalizer.fit(standardizer)

    return color_normalizer

def coordinates(batch_sample, pred_size):
    """
    To return the x, y coordinates for patch extraction
    :param dataframe batch_sample:  one entry from a dataframe
    :param int pred_size: the patch size to be predicted.
                          Here, the pred_size = 224
    :return list xylarge: x, y coordinates
    """
    xy = batch_sample.tile_loc[::-1]
    # double check
    if xy[0] == 0 or xy[1] == 0:
        xylarge = [x * pred_size for x in xy]
    else:
        xylarge = [x * pred_size - pred_size for x in xy]

    # print(batch_sample.tile_loc[::-1], xylarge)
    #img = tiles.get_tile(tiles.level_count-1, batch_sample.tile_loc[::-1])
    return xylarge

def generate_448_patches(slide, xylarge):
    """
    To extract 448x448 patches from WSI for prediction
    :param object slide: slide object from OpenSlide
    :param list xylarge: the x, y coordinates for the position where the patch
                          should be extracted
    :return array img: a 448x448 patch, only 3 channels (R, G, B)
    """
    img = slide.read_region(xylarge, 0, [448, 448])
    img = np.array(img)
    img = img[:, :, :3]

    return img

def generate_224_img_truth(slide, xylarge, truth=False):

    if truth:
        mask = truth.read_region(xylarge, 0, [224, 224])
        mask = np.array(mask)
        mask = mask[:, :, :3]

    else:

        mask = False

    img = slide.read_region(xylarge, 0, [224, 224])
    img = np.array(img)
    img = img[:, :, :3]

    return (img, mask)

def color_norm_pred(image_patch, fit, log_file, current_time):
    """
    To perform color normalization based on the method used.
    :param matrix img: the image to be color normalized
    :param object fit: the initialized method for normalization
    :return matrix img_norm: the normalized images
    :note if the color normalization fails, the original image patches
          will be used. But this event will be written in the log file.
    """
    img = image_patch
    img_norm = img
    try:
        img_standard = staintools.LuminosityStandardizer.standardize(img)
        img_norm = fit.transform(img_standard)
    except Exception as e:
        log_file.write(str(image_patch) + ';' + str(e) + ';' + current_time)
    # print(img_norm)
    return img_norm

def patch_pred_collect_from_slide_window(pred_size, fullimage, model, stride):
    """
    create a nxn matrix that includes all the patches extracted from one big patch by slide window sampling.

    :param integer pred_size: the size of patches to be extracted and predicted as tumor or normal patch.
    :param nxn matrix fullimage: the image used for slide window prediction, which is larger than the patch to be predicted to avoid side effect.
    :param object model: the trained network to predict the patches.

    :return a nxn matrix for one patch to be predicted by slide window method

    """

    output_preds_final = []

    for x in tqdm(range(0, pred_size, stride)):
               # print(x)
        patchforprediction_batch = []

        for y in range(0, pred_size, stride):
                       # for y in intervals:
                       # print(y)
            patchforprediction = fullimage[x:x+pred_size, y:y+pred_size]

            patchforprediction_batch.append(patchforprediction)

        X_train = np.array(patchforprediction_batch)

        preds = predict_batch_from_model(X_train, model)

        # print(preds)

        # output_preds.extend(preds)

        # output_preds_final.append(output_preds)
        output_preds_final.append(preds)

    output_preds_final = np.array(output_preds_final)

    return output_preds_final

def patch_pred_hnm(pred_size, fullimage, model, ground_truth, new_folder, new_slide_path, thresh_hold, xylarge):
    """
    create a nxn matrix that includes all the patches extracted from one big patch by slide window sampling.

    :param integer pred_size: the size of patches to be extracted and predicted as tumor or normal patch.
    :param nxn matrix fullimage: the image used for slide window prediction. The size here is equal to the pred_size = 224.
    :param object model: the trained network to predict the patches.

    :return a nxn matrix for one patch to be predicted by slide window method

    """

    X_train = np.expand_dims(fullimage, axis=0)

    pred = predict_batch_from_model(X_train, model)

    if np.count_nonzero(ground_truth):

        truth = 1

    else:

        truth = 0
        # update the dataframe with the new values
    if pred[0] > thresh_hold and truth == 0:

        io.imsave('%s/%s_%d_%d_hnm_%4.2f.png' % (new_folder,
                                                 osp.splitext(osp.basename(
                                                     new_slide_path))[0], xylarge[0], xylarge[1], pred[0]), fullimage)

    # print(fullimage)
    # print(pred[0])
    return pred[0]

def batch_pred_per_taskid(pred_size, stride, sub_samples, slide, fit, model, range_right, path_for_results, task_id, PatchNumber, i, j, current_time, log_file, color_norm=False):
    """
    This is the major function to do the prediction. It makes prediction for the image patches per task id.
    :param int pred_size: the size of image patch to be predicted. The default is 224.
    :param int stride: the pixels skipped by each slide window prediction.
    :param dataframe sub_sample: the part of dataframe with the patch location.
    :param object slide: the object from OpenSlide.
    :param object fit: the object to do color normalization
    :param object model: the trained neural network for prediction
    :param int range_right: the right end of the range of the sub_sample
    :param string folder_to_save: the location to save the prediction results.
    :param boolean color_norm: if or not the color normalization will be performed.

    :result array of scores for each task id. For example, 160 scores for test slides.
    """
    output_preds_final_160 = []

    for g, batch_sample in sub_samples.iterrows():

        xylarge = coordinates(batch_sample, pred_size)

        if batch_sample.is_tissue == 0:
            no_tissue_region = np.zeros(
                [int(224/stride), int(224/stride)], dtype=np.float32)
            output_preds_final_160.append(no_tissue_region)

        else:

            img = generate_448_patches(slide, xylarge)

        ####################################################################################################
        # here begins the real prediction
        ####################################################################################################
            if color_norm:
                fullimage = color_norm_pred(img, fit, log_file, current_time)
            else:
                fullimage = img

            output_preds_final = patch_pred_collect_from_slide_window(
                pred_size, fullimage, model, stride)

            # print(output_preds_final)

            output_preds_final_160.append(output_preds_final)

    output_preds_final_160 = np.array(output_preds_final_160)
    # note: don't change PatchNumber to SlideRange, i can be out of range of SlideRange dataframe
    np.save(osp.join(path_for_results, '%s_%s_%d_%d' % (format(task_id, '08'),
                                                        PatchNumber.at[i, 'SlideName'], j, range_right)), output_preds_final_160)

def batch_pred_per_taskid_hnm(pred_size, stride, sub_samples, slide, fit, model, range_right, path_for_results, task_id, PatchNumber, i, j, current_time, log_file, new_slide_path, thresh_hold, truth=False, color_norm=False):
    """
    This is the major function to do the prediction. It makes prediction for the image patches per task id.
    :param int pred_size: the size of image patch to be predicted. The default is 224.
    :param int stride: the pixels skipped by each slide window prediction.
    :param dataframe sub_sample: the part of dataframe with the patch location.
    :param object slide: the object from OpenSlide.
    :param object fit: the object to do color normalization
    :param object model: the trained neural network for prediction
    :param int range_right: the right end of the range of the sub_sample
    :param string folder_to_save: the location to save the prediction results.
    :param boolean color_norm: if or not the color normalization will be performed.

    :result array of scores for each task id. For example, 160 scores for test slides.
    """
    output_preds_final_160 = []

    for g, batch_sample in sub_samples.iterrows():

        xylarge = coordinates(batch_sample, pred_size)

        if batch_sample.is_tissue == 0:
            no_tissue_region = 0
            output_preds_final_160.append(no_tissue_region)

        else:

            img, mask = generate_224_img_truth(slide, xylarge, truth)

        ####################################################################################################
        # here begins the real prediction
        ####################################################################################################
            if color_norm:
                fullimage = color_norm_pred(img, fit, log_file, current_time)
            else:
                fullimage = img

            output_preds_final = patch_pred_hnm(
                pred_size, fullimage, model, mask, path_for_results, new_slide_path, thresh_hold, xylarge)

            # print(output_preds_final)

            output_preds_final_160.append(output_preds_final)

    output_preds_final_160 = np.array(output_preds_final_160)
    # note: don't change PatchNumber to SlideRange, i can be out of range of SlideRange dataframe
    np.save(osp.join(path_for_results, '%s_%s_%d_%d' % (format(task_id, '08'),
                                                        PatchNumber.at[i, 'SlideName'], j, range_right)), output_preds_final_160)

def exit_program(i, slide_category):
    """
    This function is only used for predicing 3d histech slides only
    :param int i: the slide index in the slide list
    :param string slide_category: can "tumor" or "normal"
    """
    if slide_category == 'normal' and (i > 99):
        sys.exit("not 3dhistech slides, program will stop")
    elif slide_category == 'tumor' and (69 < i < 109):
        sys.exit("not 3dhistech slides, program will stop")

max_stars_repo_name: DIDSR/dldp

max_issues_repo_name: DIDSR/dldp

max_forks_repo_name: DIDSR/dldp


text:
import pandas as pd
import requests
from bs4 import BeautifulSoup
import numpy as np
from time import sleep

website = lambda start, end: [f"https://en.wikipedia.org/wiki/UFC_{i}" for i in range(start, end)]

def get_top_level_data(end, start=20, get_fight_card_stats=False, both=True, avg_pause=.6):
    """

    @param end: integer
    @param start : integer
    @param get_fight_card_stats :
    @param both : boolean
    @param avg_pause : float
    """

    def add_d_set(df_use, name):
        df = df_use.copy()
        df.insert(0, "Event", f"UFC{name}")
        return df

    vals = website(start, end)
    rets_df = []
    if both:
        rets_df_2 = []

    for j, pause in zip(vals, np.random.poisson(avg_pause, len(vals))):
        print(j)
        if get_fight_card_stats and not both:
            rets_df.append(add_d_set(pd.read_html(j)[2], j))
        elif get_fight_card_stats and both:
            d = pd.read_html(j)
            rets_df.append(add_d_set(d[2], j))
            rets_df_2.append(add_d_set(d[0], j))
        else:
            rets_df.append(add_d_set(pd.read_html(j)[2], j))
            sleep(pause)
    return pd.concat(rets_df, axis=0)

max_stars_repo_name: ssunger/SportsAnalytics

max_issues_repo_name: ssunger/SportsAnalytics

max_forks_repo_name: ssunger/SportsAnalytics


text:
import os
import time

import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import f1_score, precision_score, recall_score

class Record():
    """
    This class records and stores the loss and metric value across the training and evaluation cycles.
    Further, this class prints and plots training and metric performance.
    """
    def __init__(self, is_train):
        self.is_train = is_train
        self.name = "TRAIN" if is_train else "TEST"
        self.loss = list()
        self.f1 = list()
        self.recall = list()
        self.precision = list()

    def update(self, idx, loss, y, output):
        self.loss.append(loss)
        #calculate the metrics here
        output, y = np.array(output), np.array(y)
        f1_ = f1_score(y, output, average='binary')
        precision_ = precision_score(y, output, average='binary')
        recall_ = recall_score(y, output, average='binary')
        self.f1.append(f1_)
        self.precision.append(precision_)
        self.recall.append(recall_)

        print("Epoch: %d, %s loss: %1.5f" % (idx, self.name, loss))
        print(f'{self.name} F1 score: {f1_}')
        print(f'{self.name} Precision score: {precision_}')
        print(f'{self.name} Recall: {recall_}')
        print("=" * 60)

    def get_lowest_loss(self):
        return round(min(self.loss), 4)

    def plot_losses(self, loss_values, title):
        if not self.is_train:
            plt.figure(figsize=(12, 7))
            plt.grid()
            plt.plot([x for x in self.loss], 'go--', linewidth=2, markersize=5, label='Test Loss')
            plt.plot([x for x in loss_values], 'r*-', linewidth=2, markersize=5, label='Train Loss')
            plt.xlabel('Epochs')
            plt.ylabel('Loss Values')
            plt.title(title)
            image_name = f'{title.replace(" ", "_").replace(":", "_")}_{time.strftime("%d_%m_%H_%M")}.png'
            plt.legend()
            plt.savefig(os.path.join("./routine", "plots", image_name))
        else:
            print("WARNING: CANNOT INVOKE LOSS PLOT")

    def plot_values(self, title):
        plt.figure(figsize=(12, 7))
        plt.grid()
        plt.plot([x for x in self.f1], 'go--', linewidth=2, markersize=5, label='F1 Score')
        plt.plot([x for x in self.precision], 'r*-', linewidth=2, markersize=5, label='Precision')
        plt.plot([x for x in self.recall], 'b*-', linewidth=2, markersize=5, label='Recall')
        plt.xlabel('Epochs')
        plt.ylabel('Metrics')
        plt.title(title)
        image_name = f'{title.replace(" ", "_").replace(":", "_")}_{time.strftime("%d_%m_%H_%M")}.png'
        plt.legend()
        plt.savefig(os.path.join("./routine", "plots", image_name))


text:
import os
import sys
import numpy as np

# pytorch
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, random_split, DataLoader

from scipy import stats

NITORCH_DIR = os.getcwd()
sys.path.insert(0, NITORCH_DIR)
#nitorch
from nitorch.trainer import Trainer
from nitorch.initialization import weights_init
from nitorch.metrics import *

class syntheticDataset(Dataset):
    '''A dataset of 3D synthetic data '''
    def __init__(self, n_classes=2):

        data_classes = []
        data_labels = []
        n_samples = 1000
        means = np.random.choice(n_classes, size=n_classes, replace=False)
        for c,m in zip(range(n_classes), means):
            # keep the mean at safe distances such that it can be easily seperable
            signs = np.random.choice([-1,1],size=3)
            mean = np.array([5*m, 5*m, 5*m])*signs
            covariance = np.array([[10*(m+1), 0, 0],[0, 10*(m+1), 0],[0, 0, 10*(m+1)]])
            data = np.random.multivariate_normal(mean, covariance, n_samples)
            # for labels create one-hot vectors
            # for binary classification, the label is a single value
            if(n_classes<=2):
                labels = np.tile([c],reps=(n_samples,1))
            else:
                labels = np.zeros((n_samples, n_classes))
                labels[np.arange(n_samples), c]=1

            data_classes.append(data)
            data_labels.append(labels)
        # Visualize synthetic dataset
#         fig = plt.figure()
#         ax = fig.add_subplot(111, projection='3d')
#         for i, data_class in enumerate(data_classes):
#             ax.scatter(data_class[:,0],data_class[:,1],data_class[:,2], label="Class "+str(i))
#         plt.legend()
#         plt.show()
        self.X = np.vstack(data_classes)
        self.y = np.vstack(data_labels)

    def __len__(self):
        return self.X.shape[0]

    def __getitem__(self, idx):
        return self.X[idx], self.y[idx].astype(float)

# Fully connected neural network with one hidden layer
class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size, n_classes=2):
        super(NeuralNet, self).__init__()
        self.out_size = 1 if n_classes<=2 else n_classes
        self.fc1 = nn.Linear(input_size, hidden_size).float()
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(hidden_size, self.out_size).float()

    def forward(self, x):
        out = self.fc1(x)
        out = self.relu(out)
        out = self.fc2(out)
        if(self.out_size >1):
            out = F.softmax(out)
        else:
            out = torch.sigmoid(out)
        return out

if __name__ == "__main__":

    print("Starting tests ...")
    if(torch.cuda.is_available()):
        device = torch.device('cpu')
        print("Running on GPU ...")
    else:
        device = torch.device('cpu')
        print("Running on CPU ...")

    #############################################
    # Test 1 : Binary classification
    #############################################
    print("-----------------------------------------------\nTesting nitorch.trainer.Trainer() : Binary classification")
    n_classes = 2
    BATCH_SIZE = 64
    EPOCHS = 20
#     random.seed(42)
#     np.random.seed(42)
#     torch.manual_seed(42)

    data = syntheticDataset(n_classes=n_classes)
    # shuffle and split into test and train
    train_size = int(0.75*len(data))
    train_data, val_data = random_split(data, (train_size, len(data) - train_size))
    train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)
    val_loader = DataLoader(val_data, batch_size=BATCH_SIZE, shuffle=True)

    # NETWORK
    net = NeuralNet(3, 10).to(device).double()
    net.apply(weights_init)
    criterion = nn.BCELoss().to(device)
    optimizer = optim.Adam(net.parameters(),
                           lr=1e-3,
                           weight_decay=1e-5)

    metrics = [specificity, sensitivity, binary_balanced_accuracy]

    trainer = Trainer(
    net,
    criterion,
    optimizer,
    # scheduler=None,
    metrics=metrics,
    # callbacks=callbacks,
    device=device,
    prediction_type="binary")

    # train model and store results
    net, report = trainer.train_model(
        train_loader,
        val_loader,
        num_epochs=EPOCHS,
        show_train_steps=5,
        show_validation_epochs=5
        )

    # Test 1-a: Check if the loss reduces with
    # a negative slope on the valdation dataset
    y = (report["val_metrics"]["loss"])
    x = np.arange(len(y))
    slope, intercept, _, _, _ = stats.linregress(x,y)
    # plt.scatter(x, y)
    # plt.plot((slope*x + intercept))
    # plt.show()
    assert slope<0, "Test 1-a 'binary classification': The loss is not reducing with training - TEST FAILED"
    print("Test 1-a 'binary classification': The loss on the valdation dataset reduces with training - TEST PASSED")

    # Test 1-b: Check if the metrics are calculated correctly and improve with training
    assert (report["val_metrics"]['binary_balanced_accuracy'][0]<report["val_metrics"]['binary_balanced_accuracy'][-1]) or (report["val_metrics"]['binary_balanced_accuracy'][0] < report["val_metrics"]['binary_balanced_accuracy'][-2]),  "Test 1-b : The accuracy did not improve with training - TEST FAILED"
    print("Test 1-b 'binary classification': 'binary_balanced_accuracy' metric improved with training - TEST PASSED")

     #############################################
    # Test 2 : Multi-class classification
    #############################################
    print("-----------------------------------------------\nTesting nitorch.trainer.Trainer() : Multi-class classification")
    n_classes = 4
    data = syntheticDataset(n_classes=n_classes)

    # shuffle and split into test and train
    train_size = int(0.75*len(data))
    train_data, val_data = random_split(data, (train_size, len(data) - train_size))
    train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True)
    val_loader = DataLoader(val_data, batch_size=BATCH_SIZE, shuffle=True)

    # NETWORK
    net = NeuralNet(3, 10, n_classes=n_classes).to(device).double()
    net.apply(weights_init)
    criterion = nn.BCELoss().to(device)
    optimizer = optim.Adam(net.parameters(),
                           lr=1e-3,
                           weight_decay=1e-5)

    metrics = [classif_accuracy]

    trainer = Trainer(
    net,
    criterion,
    optimizer,
    # scheduler=None,
    metrics=metrics,
    # callbacks=callbacks,
    device=device,
    prediction_type="classification")

    # train model and store results
    net, report = trainer.train_model(
        train_loader,
        val_loader,
        num_epochs=EPOCHS,
        show_train_steps=5,
        show_validation_epochs=5
        )
    # Test 2-a: Check if the loss reduces with a negative slope on the valdation dataset
    y = (report["val_metrics"]["loss"])
    x = np.arange(len(y))
    slope, intercept, _, _, _ = stats.linregress(x,y)
    # plt.scatter(x, y)
    # plt.plot((slope*x + intercept))
    # plt.show()
    assert slope<0, "Test 1-a 'Multi-class classification': The loss is not reducing with training - TEST FAILED"
    print("Test 1-a 'Multi-class classification': The loss on the valdation dataset reduces with training - TEST PASSED")

    # Test 2-b: Check if the metrics are calculated correctly and improve with training
    assert (report["val_metrics"]['classif_accuracy'][0]<report["val_metrics"]['classif_accuracy'][-1]) or (report["val_metrics"]['classif_accuracy'][0] < report["val_metrics"]['binary_balanced_accuracy'][-2]),  "Test 1-b : The accuracy did not improve with training - TEST FAILED"
    print("Test 1-b 'Multi-class classification': 'classif_accuracy' metric improved with training - TEST PASSED")

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-

# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the MIT License.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# MIT License for more details.

"""This is a class for coco dataset."""
import logging
import os
import json
import numpy as np
import torch
from PIL import Image
from zeus.common import ClassFactory, ClassType
from zeus.datasets.conf.coco import CocoConfig
from zeus.datasets.common.utils.dataset import Dataset
from pycocotools import mask as coco_mask
from pycocotools.coco import COCO
from zeus.common.task_ops import TaskOps

@ClassFactory.register(ClassType.DATASET)
class CocoDataset(Dataset):
    """This is the class of coco dataset, which is a subclass of Dateset.

    :param train: `train`, `val` or test
    :type train: str
    :param cfg: the config the dataset need, defaults to None, and if the cfg is None,
    the default config will be used, the default config file is a yml file with the same name of the class
    :type cfg: yml, py or dict
    """

    config = CocoConfig()

    def __init__(self, **kwargs):
        """Construct the CocoDataset class."""
        super(CocoDataset, self).__init__(**kwargs)
        self.dataset_init()

    def dataset_init(self):
        """Construct method."""
        self.num_classes = self.args.num_classes
        self.data_root = self.args.data_root
        img_prefix = str(self.args.img_prefix)
        ann_prefix = self.args.ann_prefix
        self.ann_file = os.path.join(self.data_root, 'annotations',
                                     '{}_{}{}.json'.format(ann_prefix, self.mode, img_prefix))
        self.img_dir = os.path.join(self.data_root, '{}{}'.format(self.mode, img_prefix))
        self.coco = COCO(self.ann_file)
        self.ids = list(sorted(self.coco.imgs.keys()))
        # filter invalid data
        self.ids = list(filter(self._filter_invalid_ids, self.ids))
        self.collate_fn = collate_fn

    def __getitem__(self, index):
        """Get an item of the dataset according to the index.

        :param index: index
        :type index: int
        :return: an item of the dataset according to the index
        :rtype: dict
        """
        img_id = self.ids[index]
        ann_ids = self.coco.getAnnIds(imgIds=img_id)
        target = self.coco.loadAnns(ann_ids)
        file_name = self.coco.loadImgs(img_id)[0]['file_name']
        img_path = os.path.join(self.img_dir, file_name)
        img = Image.open(img_path).convert('RGB')
        target = dict(image_id=img_id, annotations=target)
        if self.transforms is not None:
            img, target = self.transforms(img, target)
        return img, target

    def __len__(self):
        """Get the length of the dataset."""
        return len(self.ids)

    def _filter_invalid_ids(self, img_id):
        ann_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=0)
        anno = self.coco.loadAnns(ann_ids)
        return self._has_valid_annotation(anno)

    def _has_valid_annotation(self, anno):
        min_keypoints_per_image = 10

        def _has_only_empty_bbox(anno):
            return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno)

        # if it's empty, there is no annotation
        def _count_visible_keypoints(anno):
            return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno)

        if len(anno) == 0:
            return False
        if _has_only_empty_bbox(anno):
            return False
        if "keypoints" not in anno[0]:
            return True
        if _count_visible_keypoints(anno) >= min_keypoints_per_image:
            return True
        return False

def collate_fn(batch):
    """Collate fn for data loader."""
    return tuple(zip(*batch))

@ClassFactory.register(ClassType.DATASET)
class DetectionDataset(Dataset):
    """Detection common dataset."""

    config = CocoConfig()

    def __init__(self, **kwargs):
        """Construct the Detection Dataset class."""
        super(DetectionDataset, self).__init__(**kwargs)
        self.imgs = list(sorted(os.listdir(os.path.join(self.args.data_root, self.args.img_prefix))))
        self.masks = list(sorted(os.listdir(os.path.join(self.args.data_root, self.args.ann_prefix))))
        portion = self.args.test_size
        self.imgs = self.imgs[:-portion] if self.mode == 'train' else self.imgs[-portion:]
        self.masks = self.masks[:-portion] if self.mode == 'train' else self.masks[-portion:]
        self.collate_fn = collate_fn
        convert_to_coco_api(self)

    def __getitem__(self, idx):
        """Get an item of the dataset according to the index."""
        # load images and masks
        img_path = os.path.join(self.args.data_root, self.args.img_prefix, self.imgs[idx])
        mask_path = os.path.join(self.args.data_root, self.args.ann_prefix, self.masks[idx])
        img = Image.open(img_path).convert("RGB")
        mask = Image.open(mask_path)
        mask = np.array(mask)
        obj_ids = np.unique(mask)
        obj_ids = obj_ids[1:]
        masks = mask == obj_ids[:, None, None]
        num_objs = len(obj_ids)
        boxes = []
        for i in range(num_objs):
            pos = np.where(masks[i])
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, ymin, xmax, ymax])
        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        labels = torch.ones((num_objs,), dtype=torch.int64)
        masks = torch.as_tensor(masks, dtype=torch.uint8)
        image_id = torch.tensor([idx])
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
        iscrowd = torch.zeros((num_objs,), dtype=torch.int64)

        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["masks"] = masks
        target["image_id"] = image_id
        target["area"] = area
        target["iscrowd"] = iscrowd

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target

    def __len__(self):
        """Get the length of the dataset."""
        return len(self.imgs)

def convert_to_coco_api(ds):
    """Convert to coco dataset."""
    coco_ds = COCO()
    ann_id = 1
    dataset = {'images': [], 'categories': [], 'annotations': []}
    categories = set()
    for img_idx in range(len(ds)):
        img, targets = ds[img_idx]
        image_id = targets["image_id"].item()
        img_dict = {}
        img_dict['id'] = image_id
        img_dict['height'] = img.shape[-2]
        img_dict['width'] = img.shape[-1]
        dataset['images'].append(img_dict)
        bboxes = targets["boxes"]
        bboxes[:, 2:] -= bboxes[:, :2]
        bboxes = bboxes.tolist()
        labels = targets['labels'].tolist()
        areas = targets['area'].tolist()
        iscrowd = targets['iscrowd'].tolist()
        if 'masks' in targets:
            masks = targets['masks']
            masks = masks.permute(0, 2, 1).contiguous().permute(0, 2, 1)
        if 'keypoints' in targets:
            keypoints = targets['keypoints']
            keypoints = keypoints.reshape(keypoints.shape[0], -1).tolist()
        num_objs = len(bboxes)
        for i in range(num_objs):
            ann = {}
            ann['image_id'] = image_id
            ann['bbox'] = bboxes[i]
            ann['category_id'] = labels[i]
            categories.add(labels[i])
            ann['area'] = areas[i]
            ann['iscrowd'] = iscrowd[i]
            ann['id'] = ann_id
            if 'keypoints' in targets:
                ann['keypoints'] = keypoints[i]
                ann['num_keypoints'] = sum(k != 0 for k in keypoints[i][2::3])
            dataset['annotations'].append(ann)
            ann_id += 1
    dataset['categories'] = [{'id': i} for i in sorted(categories)]
    coco_ds.dataset = dataset
    coco_ds.createIndex()
    instances_val = os.path.join(TaskOps().local_output_path, 'instances.json')
    json.dump(coco_ds.dataset, open(instances_val, 'w'))
    logging.info("dump detection instances json file: {}".format(instances_val))
    return coco_ds

max_stars_repo_name: georgefang/vega

max_issues_repo_name: georgefang/vega

max_forks_repo_name: georgefang/vega


text:
import math
import string
import numpy as np

from heapq import heappush, heappop

def get_random_string(length=24):
    letters = string.ascii_lowercase
    result_str = ''.join(np.random.choice([char for char in letters]) for _ in range(length))
    return result_str

def read(fn):
    s = open(fn, 'r').read()
    return eval(s)

def getCDFValue(cdf, key):
    for x in sorted(cdf.keys()):
        if key < float(x):
            return cdf[x]
    return None

def getCDFAverage(cdf):
    average = 0
    previous_x = 0
    for x in sorted(cdf.keys()):
        average += cdf[x]*(float(x)-previous_x)
        previous_x = float(x)
    return average

class EventsManager():

    def __init__(self, cdf_path='hadoop/', connection_target=10000, update_rate=10):

        self.connection_duration_cdf = read(cdf_path + 'data/hadoop_cdf.txt')
        self.server_updates_cdf = read(cdf_path + 'data/downtime_upgrade_cdf.txt')
        self.connection_size_cdf = read(cdf_path + 'data/hadoop_size_cdf.txt')

        # the default values by CHEETAH
        self.connection_target = connection_target
        self.update_rate = update_rate
        self.connection_rate = self.connection_target / getCDFAverage(self.connection_duration_cdf)

        # event heap
        self.events = [(0, "new-connection"), (0, "kil-server")]

        self.flows = set()
        self.flows_data = {}
        self.flows_spec = {}

    def next(self):

        time, event = heappop(self.events)

        if "new-connection" in event:
            conn_id = get_random_string()
            while conn_id in self.flows:
                conn_id = get_random_string()
            self.flows.add(conn_id)

            connection_duration = getCDFValue(self.connection_duration_cdf, np.random.random())
            heappush(self.events, (time + connection_duration, "del-connection" + " " + conn_id))

            # assume cdf in KB and we have ~100B in a packet
            connection_mass = int(10*getCDFValue(self.connection_size_cdf, np.random.random())) + 1.0
            self.flows_data[conn_id] = (connection_mass, time, time + connection_duration)
            self.flows_spec[conn_id] = (connection_mass, time, time + connection_duration)

            connection_next = -math.log(1-np.random.random(), math.e) / self.connection_rate
            heappush(self.events, (time + connection_next, "new-connection"))

            return time, "new-connection", conn_id

        if "del-connection" in event:

            conn_id = event.split(" ")[1]

            try:  # use del_spec_coneciton to clean self.flows_spec[conn]
                self.flows.remove(conn_id)
                del self.flows_data[conn_id]
                return time, "del-connection", conn_id
            except:  # assosiated server is allready removed
                return self.next()

        if event == "kil-server":
            update_ends = getCDFValue(self.server_updates_cdf, np.random.random())
            heappush(self.events,(time + update_ends, "new-server"))
            update_next = -math.log(1 - np.random.random(), math.e)/(self.update_rate/60)
            heappush(self.events,(time + update_next, "kil-server"))
            return time, "kil-server", None

        if event == "new-server":
            return time, "new-server", None

        raise Exception("cannot arrive here: {}:{}".format(time, event))

    def get_params(self):
        return self.connection_target, self.update_rate

    def get_active_connections(self, time):

        active_flows = []
        for flow in self.flows:
            conn_mass, conn_start_time, conn_end_time = self.flows_data[flow]
            passed_time = time - conn_start_time
            if passed_time == 0:
                continue
            elif passed_time < 0:
                raise Exception("negative time")

            passed_conn_time_fraction = (time - conn_start_time) / (conn_end_time - conn_start_time)
            if passed_conn_time_fraction > 1 or passed_conn_time_fraction < 0:
                raise Exception("illegal time")

            passed_conn_packets = np.random.binomial(n=conn_mass, p=passed_conn_time_fraction)
            if passed_conn_packets > 0:
                active_flows.append(flow)

            self.flows_data[flow] = conn_mass - passed_conn_packets, time, conn_end_time

        print("get_active_connections: sampled {} out of {}.".format(len(active_flows), len(self.flows)))

        return active_flows

    def del_stale_conecitons(self, conn_list):
        self.flows -= conn_list
        for f in conn_list:
            del self.flows_data[f]
            del self.flows_spec[f]

    def del_spec_coneciton(self, conn):
        del self.flows_spec[conn]

if __name__ == "__main__":
    m = EventsManager()
    for i in range(1000):
        time, revent, misc = m.next()
        print(time, revent, misc)

max_stars_repo_name: anchorhash/jetlb

max_issues_repo_name: anchorhash/jetlb

max_forks_repo_name: anchorhash/jetlb


text:
import sys
import pathlib
import time

import tqdm
import numpy as np
import torch
import torch.nn.functional
import torch.utils.data
import matplotlib.pyplot as plt

import dset_spaces.dset1
import dset_realestate.dset1
import dset_blender.dset1
import model_deepview
import vgg
import misc
import utils_render
import test_engine

########################################################################################################################
class TrainerDeepview:
    """
    Trainer class for "mini_deep_view_no_lgd"
    """

    def __init__(self, dset_dir, dset_name, dset_options={}, device=torch.device('cuda'), lr=1.e-3, batch_size=1,
                 im_w=200, im_h=200, borders=(0, 0)):
        self.device = device
        self.batch_size = batch_size
        self.lr = lr
        self.num_planes = 10
        self.num_workers = 4
        self.borders = borders
        # Model
        self.model = model_deepview.DeepViewLargeModel().to(device=device)
        # VGG loss
        self.vgg_loss = vgg.VGGPerceptualLoss1(resize=False, device=device)
        # Dataset+loaders
        print(
            f'TrainerDeepview: dset_dir={dset_dir}, dset_name={dset_name}, dset_options={dset_options}, device={device}')
        if dset_name == 'spaces:1deterministic':
            self.dset_train = dset_spaces.dset1.DsetSpaces1(dset_dir, False, im_w=im_w, im_h=im_h, **dset_options)
            self.dset_val = dset_spaces.dset1.DsetSpaces1(dset_dir, True, im_w=im_w, im_h=im_h, **dset_options)
        elif dset_name == 're:1random':
            self.dset_train = dset_realestate.dset1.DsetRealEstate1(dset_dir, False, im_w=im_w, im_h=im_h,
                                                                    **dset_options)
            self.dset_val = dset_realestate.dset1.DsetRealEstate1(dset_dir, True, im_w=im_w, im_h=im_h, **dset_options)
        elif dset_name =="blender":
            self.dset_train = dset_blender.dset1.DsetBlender(dset_dir, False, im_w=im_w, im_h=im_h,
                                                                    **dset_options)
            self.dset_val = dset_blender.dset1.DsetBlender(dset_dir, True, im_w=im_w, im_h=im_h, **dset_options)
        else:
            raise ValueError(f'Wrong dset_name={dset_name} !')

        print(f'Datasets : train: {len(self.dset_train)},  val: {len(self.dset_val)}')
        self.loader_train = torch.utils.data.DataLoader(self.dset_train, batch_size=self.batch_size,
                                                        num_workers=self.num_workers, shuffle=True)
        self.loader_val = torch.utils.data.DataLoader(self.dset_val, batch_size=self.batch_size,
                                                      num_workers=self.num_workers)
        # Optimizer
        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)

        if False:
            b = next(iter(self.loader_train))
            print(('KEYS=', list(b.keys())))
            for k, v in list(b.items()):
                print(f'{k} : {v.shape} {v.dtype}')
            sys.exit(0)

    def process_one(self, out, x):
        """Process and render one network output+target"""
        out = torch.sigmoid(out)
        rgba_layers = out.permute(0, 3, 4, 1, 2)

        # print('SHAPE=', rgba_layers.shape)
        # print(f'MIN={rgba_layers.min().item()}, MAX={rgba_layers.max().item()}')
        n_targets = x['tgt_img'].shape[1]

        # t1 = time.time()
        batch_size = rgba_layers.shape[0]
        # i_batch = 0  # Replace later with a loop over batch
        out_images_batch = []
        # Can we fully batch the second version, I wonder?
        for i_batch in range(batch_size):
            if False:
                # Version with a loop over targets
                outs = []
                for i_target in range(n_targets):
                    rel_pose = torch.matmul(x['tgt_cfw'][i_batch, i_target, :, :], x['ref_wfc'][i_batch]).unsqueeze(0)
                    intrin_tgt = x['tgt_intrin'][i_batch, i_target, :, :].unsqueeze(0)
                    intrin_ref = x['ref_intrin'][i_batch].unsqueeze(0)
                    out_image = utils_render.mpi_render_view_torch(rgba_layers, rel_pose, x['mpi_planes'][i_batch],
                                                                   intrin_tgt, intrin_ref)
                    outs.append(out_image)
                out_images = torch.cat(outs, 0)
            else:
                # Version batched over targets, but we have to repeat a relatively large tensor rgba, which one is better?
                # Results are very close (but not to machine precision !)
                rel_pose = torch.matmul(x['tgt_cfw'][i_batch], x['ref_wfc'][i_batch])
                intrin_tgt = x['tgt_intrin'][i_batch]
                intrin_ref = x['ref_intrin'][i_batch].unsqueeze(0).repeat(n_targets, 1, 1)
                rgba = rgba_layers[i_batch].unsqueeze(0).repeat(n_targets, 1, 1, 1, 1)

                out_images = utils_render.mpi_render_view_torch(rgba, rel_pose, x['mpi_planes'][i_batch], intrin_tgt, intrin_ref)

            # t2 = time.time()
            # print('TIME RENDER', t2-t1)
            out_images_batch.append(out_images)
        out_images_batch = torch.cat(out_images_batch, dim=0)
        targets = x['tgt_img']
        targets = targets.reshape(batch_size * n_targets, *targets.shape[2:])
        return out_images_batch, targets

    def loss(self, out, x):
        """Calculate VGG loss"""
        output_image, target = self.process_one(out, x)
        output_image = misc.my_crop(output_image, self.borders)
        target = misc.my_crop(target, self.borders)
        loss = self.vgg_loss(output_image, target)
        return loss

    def val(self):
        """Validate, 1 run"""
        self.model.eval()
        loss_sum = 0
        with torch.no_grad():
            for batch in self.loader_val:
                x = misc.to_device(batch, self.device)
                out = self.model(x)
                loss = self.loss(out, x)
                loss_sum += loss.item()
        return loss_sum / len(self.loader_val)

    def train(self):
        """Train 1 epoch"""
        self.model.train()
        loss_sum = 0
        for batch in tqdm.tqdm(self.loader_train):
            x = misc.to_device(batch, self.device)
            # t1 = time.time()
            self.optimizer.zero_grad()
            out = self.model(x)
            loss = self.loss(out, x)
            loss.backward()
            self.optimizer.step()
            loss_sum += loss.item()
            # t2 = time.time()
            # print('TIME TRAIN RUN', t2-t1)
        return loss_sum / len(self.loader_train)

    def train_loop(self, n_epoch=10):
        """Train the model"""
        for i_epoch in range(n_epoch):
            t1 = time.time()
            loss_train = self.train()
            loss_val = self.val()
            t2 = time.time()
            print(
                f'\nEpoch {i_epoch}/{n_epoch} : loss_train = {loss_train}, loss_val = {loss_val}, time = {t2 - t1:6.2f}s')
        # Save the trained model
        self.save_model()

    def save_model(self, filepath_s='./trained-models/dview.pt'):
        filepath = pathlib.Path(filepath_s)
        if not filepath.parent.exists():
            filepath.parent.mkdir()
        torch.save(self.model.state_dict(), str(filepath))

    def load_model(self, filepath_s='./trained-models/dview.pt'):
        filepath = pathlib.Path(filepath_s)
        if filepath.exists():
            self.model.load_state_dict(torch.load(str(filepath)))

    def demo_draw(self):
        """Draw predictions vs targets as a test"""
        self.model.eval()
        batch = next(iter(self.loader_val))
        x = misc.to_device(batch, self.device)
        with torch.no_grad():
            out = self.model(x)
        t1, t2 = self.process_one(out, x)
        nt = t1.shape[0]
        nt = min(nt, 1)
        for i in range(nt):
            plt.subplot(nt, 2, 2 * i + 1)
            plt.imshow(misc.tens2rgb(t1[i]))
            plt.axis('off')
            plt.subplot(nt, 2, 2 * i + 2)
            plt.imshow(misc.tens2rgb(t2[i]))
            plt.axis('off')
        plt.tight_layout()
        plt.show()

    def create_html_viewer(self, scene_idx=0):
        """Infer a batch, and create an HTML viewer from the template"""
        self.model.eval()

        # Choose your scene_idx from the val set !
        # Dataloaders like this adds batch dimension to a chosen element !
        scene_loader = torch.utils.data.DataLoader(self.dset_val, batch_sampler=[[scene_idx]])
        x = misc.to_device(next(iter(scene_loader)), self.device)  # One batch

        with torch.no_grad():
            out = self.model(x)
        out = torch.sigmoid(out)
        rgba_layers = out.permute(0, 3, 4, 1, 2)

        # By now we have RGBA MPI in the [0, 1] range
        # Export them to the HTML
        p_viewer = pathlib.Path('generated-html')
        if not p_viewer.exists():
            p_viewer.mkdir()
        # print(rgba_layers.shape, rgba_layers.dtype)
        for i in range(self.num_planes):
            layer = i
            file_path = 'generated-html/mpi{}.png'.format(("0" + str(layer))[-2:])
            img = rgba_layers[0, :, :, layer, :]
            misc.save_image(img, file_path)

        image_srcs = [misc.get_base64_encoded_image('./generated-html/mpi{}.png'.format(("0" + str(i))[-2:])) for i in
                      range(self.num_planes)]

        with open("./deepview-mpi-viewer-template.html", "r") as template_file:
            template_str = template_file.read()

        MPI_SOURCES_DATA = ",".join(['\"' + img_src + '\"' for img_src in image_srcs])
        template_str = template_str.replace("const mpiSources = MPI_SOURCES_DATA;",
                                            "const mpiSources = [{}];".format(MPI_SOURCES_DATA))

        with open("./generated-html/deepview-mpi-viewer.html", "w") as output_file:
            output_file.write(template_str)

    def test(self):
        """Test on the val set with the test engine (SSIM)"""
        self.model.eval()
        engine = test_engine.TestEngine()
        with torch.no_grad():
            for batch in tqdm.tqdm(self.loader_val):
                x = misc.to_device(batch, self.device)
                out = self.model(x)
                outs, targets = self.process_one(out, x)
                engine.run_batch(x, outs, targets, self.borders)
        engine.print_stats()

########################################################################################################################

max_stars_repo_name: hturki/deepview

max_issues_repo_name: hturki/deepview

max_forks_repo_name: hturki/deepview