text:
def addr_shape(shapetype):
    #Process user-defined shapefile from DC OpenData

    import numpy as np
    import pandas as pd
    import geopandas as gpd

    assert shapetype == 'census' or 'ward' or 'anc' #Currently only supports these

    crs='EPSG:4326' #Convenience assignment of crs throughout

    if shapetype == 'census':
        shp_fl = down_extract_zip(
            'https://opendata.arcgis.com/datasets/6969dd63c5cb4d6aa32f15effb8311f3_8.zip'
        ) #Download the zip file and extract it, then assign the shapefile path
        shp_census = gpd.read_file(shp_fl, crs=crs)
        shp_df     = gpd.GeoDataFrame(shp_census,
                                  crs=crs,
                                  geometry=shp_census['geometry']
                                 )
        shp_df = shp_df.rename(columns={'TRACT': 'NAME'}) #Fix header

    if shapetype == 'ward':
        shp_fl = down_extract_zip(
            'https://opendata.arcgis.com/datasets/0ef47379cbae44e88267c01eaec2ff6e_31.zip'
        ) #Download the zip file and extract it, then assign the shapefile path
        shp_ward   = gpd.read_file(shp_fl, crs=crs)
        shp_df     = gpd.GeoDataFrame(shp_ward,
                                  crs=crs,
                                  geometry=shp_ward['geometry']
                                 )

    if shapetype == 'anc':
        shp_fl = down_extract_zip(
            'https://opendata.arcgis.com/datasets/fcfbf29074e549d8aff9b9c708179291_1.zip'
        ) #Download the zip file and extract it, then assign the shapefile path
        shp_anc    = gpd.read_file(shp_fl, crs=crs)
        shp_df     = gpd.GeoDataFrame(shp_anc,
                                  crs=crs,
                                  geometry=shp_anc['geometry']
                                 )

    adr_df  =pd.read_csv(
      './data/Address_Points.csv', encoding='utf-8', low_memory=False)

    return [adr_df, shp_df]

def down_extract_zip(url):
    #Downloads and unzips a DCopendata shape file
    #and returns the filepath of the shape file

    #Usage: must have file named 'data' in cwd.
    #Then: flname = down_extract_zip(url_of_zipfile)
    #flname is now the path of the shpfile.

    import os, zipfile, requests, time

    local_filename = str('./data/' + url.split('/')[-1])

    r = requests.get(url)

    while r.status_code != 200:
        print('Connection error: retrying...')
        time.sleep(10)
        r = requests.get(url)

    assert r.status_code == 200 #Check connection
    with open(local_filename, 'wb') as f:
        f.write(r.content)
    zip_fld = local_filename[:-4]
    if not os.path.exists(zip_fld):
        os.makedirs(zip_fld)
        with zipfile.ZipFile(local_filename, "r") as zip_pl:
            zip_pl.extractall(zip_fld)
    fld_arr = os.listdir(zip_fld) #get array of unzipped files
    for unzippedfile in fld_arr:  #find shapefile in unzipped files
        if str(unzippedfile[-3:]) == 'shp':
            flname = zip_fld + '/' + unzippedfile
    if not flname:
        print('Shapefile not found!')
    return flname


text:
#!/usr/bin/env python
# coding: utf-8
#es_host = 'localhost:9200'
import csv
import gzip
import json
import logging
import os
import tarfile
import time
from datetime import datetime
#from es_pandas import es_pandas
from functools import reduce
from glob import glob
from typing import Callable, Dict, Iterable, List, Type

import numpy as np
import pandas as pd
import seaborn as sns
import torch
import transformers
#from elasticsearch import Elasticsearch, helpers
from sentence_transformers import (InputExample, LoggingHandler,
                                   SentenceTransformer, util)
from sentence_transformers.cross_encoder import CrossEncoder
from sentence_transformers.cross_encoder.evaluation import CERerankingEvaluator
from sentence_transformers.evaluation import (SentenceEvaluator,
                                              SequentialEvaluator)
from torch import nn
from torch.optim import Optimizer
from torch.utils.data import DataLoader
from tqdm.autonotebook import tqdm, trange
from transformers import (AutoConfig, AutoModelForSequenceClassification,
                          AutoTokenizer)

logger = logging.getLogger(__name__)
pd.set_option("display.max_rows", 200)

class DictionaryEvaluator(SequentialEvaluator):
    def __init__(self, evaluators: Iterable[SentenceEvaluator], main_score_function = lambda x: x):
        super(DictionaryEvaluator, self).__init__(evaluators, main_score_function)
        #self.eval_name_ext_dict = dict(map(lambda t2: (t2[0].lower()[:t2[0].lower().find("Evaluator".lower())], t2[1]) ,map(lambda eval_ext: (eval_ext.name, eval_ext) ,self.evaluators)))
        self.eval_name_ext_dict = dict(map(lambda t2: (t2[0], t2[1]) ,map(lambda eval_ext: (eval_ext.name, eval_ext) ,self.evaluators)))
    def __call__(self, model, output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
        scores = {}
        #for evaluator in self.evaluators:
        for eval_ext_name, evaluator in self.eval_name_ext_dict.items():
            #scores.append(evaluator(model, output_path, epoch, steps))
            #eval_output_path = output_path + eval_ext_name
            eval_output_path = output_path + "_" + eval_ext_name
            #scores[eval_ext_name] = evaluator(model, output_path, epoch, steps)
            if eval_output_path is not None and not(os.path.exists(eval_output_path)):
                os.makedirs(eval_output_path, exist_ok=True)
            scores[eval_ext_name] = evaluator(model, eval_output_path, epoch, steps)
        return self.main_score_function(scores)

class CrossEncoder_Dict_Eval(CrossEncoder):
    def _eval_during_training(self, evaluator, output_path, save_best_model, epoch, steps, callback):
        """Runs evaluation during the training"""
        if evaluator is not None:
            if isinstance(evaluator, DictionaryEvaluator):
                if type(self.best_score) != type({}):
                    self.best_score = dict(map(lambda eval_ext_name: (eval_ext_name, self.best_score), evaluator.eval_name_ext_dict.keys()))
                score_dict = evaluator(self, output_path=output_path, epoch=epoch, steps=steps)
                if callback is not None:
                    callback(score, epoch, steps)
                for eval_ext_name, eval_score in score_dict.items():
                    if eval_score > self.best_score[eval_ext_name]:
                        self.best_score[eval_ext_name] = eval_score
                        if save_best_model:
                            #eval_output_path = output_path + eval_ext_name
                            eval_output_path = output_path + "_" + eval_ext_name
                            self.save(eval_output_path)
            else:
                score = evaluator(self, output_path=output_path, epoch=epoch, steps=steps)
                if callback is not None:
                    callback(score, epoch, steps)
                if score > self.best_score:
                    self.best_score = score
                    if save_best_model:
                        self.save(output_path)

class CERerankingEvaluatorSUM:
    """
    This class evaluates a CrossEncoder model for the task of re-ranking.

    Given a query and a list of documents, it computes the score [query, doc_i] for all possible
    documents and sorts them in decreasing order. Then, MRR@10 is compute to measure the quality of the ranking.

    :param samples: Must be a list and each element is of the form: {'query': '', 'positive': [], 'negative': []}. Query is the search query,
     positive is a list of positive (relevant) documents, negative is a list of negative (irrelevant) documents.
    """
    def __init__(self, samples, mrr_at_k: int = 10, name: str = '', num_dev_queries = 600):
        self.samples = samples
        self.name = name
        self.mrr_at_k = mrr_at_k

        self.num_dev_queries = num_dev_queries

        if isinstance(self.samples, dict):
            self.samples = list(self.samples.values())
        #output/training_ms-marco_cross-encoder-xlm-roberta-base-2021-01-12_21-10-39mrr-train-eva/CERerankingEvaluator_mrr-train-eval_results.csv
        self.csv_file = "CERerankingEvaluator" + ("_" + name if name else '') + "_results.csv"
        self.csv_headers = ["epoch", "steps", "MRR@{}".format(mrr_at_k)]

        self.score_json_file = self.csv_file.replace(".csv", ".json")

    def __call__(self, model, output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
        if epoch != -1:
            if steps == -1:
                out_txt = " after epoch {}:".format(epoch)
            else:
                out_txt = " in epoch {} after {} steps:".format(epoch, steps)
        else:
            out_txt = ":"

        logger.info("CERerankingEvaluator: Evaluating the model on " + self.name + " dataset" + out_txt)

        all_mrr_scores = []
        num_queries = 0
        num_positives = []
        num_negatives = []
        scores_list = []

        samples = list(self.samples)
        samples_indexes = np.random.permutation(np.arange(len(samples)))
        samples = list(map(lambda idx: samples[idx], samples_indexes[:self.num_dev_queries]))
        #for instance in self.samples:
        for instance in samples:
            query = instance['query']
            positive = list(instance['positive'])
            negative = list(instance['negative'])
            docs = positive + negative
            is_relevant = [True]*len(positive) + [False]*len(negative)

            if len(positive) == 0 or len(negative) == 0:
                continue

            num_queries += 1
            num_positives.append(len(positive))
            num_negatives.append(len(negative))

            model_input = [[query, doc] for doc in docs]
            pred_scores = model.predict(model_input, convert_to_numpy=True, show_progress_bar=False)
            scores_list.extend(list(pred_scores))
            pred_scores_argsort = np.argsort(-pred_scores)  #Sort in decreasing order

            mrr_score = 0
            for rank, index in enumerate(pred_scores_argsort[0:self.mrr_at_k]):
                if is_relevant[index]:
                    mrr_score += 1 / (rank+1)

            all_mrr_scores.append(mrr_score)

        mean_mrr = np.mean(all_mrr_scores)
        logger.info("Queries: {} \t Positives: Min {:.1f}, Mean {:.1f}, Max {:.1f} \t Negatives: Min {:.1f}, Mean {:.1f}, Max {:.1f}".format(num_queries, np.min(num_positives), np.mean(num_positives), np.max(num_positives), np.min(num_negatives), np.mean(num_negatives), np.max(num_negatives)))
        logger.info("MRR@{}: {:.2f}".format(self.mrr_at_k, mean_mrr*100))

        if output_path is not None:
            csv_path = os.path.join(output_path, self.csv_file)
            output_file_exists = os.path.isfile(csv_path)
            with open(csv_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
                writer = csv.writer(f)
                if not output_file_exists:
                    writer.writerow(self.csv_headers)
                writer.writerow([epoch, steps, mean_mrr])
            json_path = os.path.join(output_path, self.score_json_file)
            output_file_exists = os.path.isfile(json_path)
            with open(json_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
                writer = csv.writer(f)
                if not output_file_exists:
                    writer.writerow(["epoch", "steps", "score@"])
                writer.writerow([epoch, steps, json.dumps({"scores_list": list(map(float ,scores_list))})])
        return mean_mrr

class ScoreCalculator(object):
    def __init__(self,
                queries_ids,
                relevant_docs,
                mrr_at_k: List[int] = [10],
                ndcg_at_k: List[int] = [10],
                accuracy_at_k: List[int] = [1, 3, 5, 10],
                precision_recall_at_k: List[int] = [1, 3, 5, 10],
                map_at_k: List[int] = [100],
    ):
        "queries_ids list of query, relevant_docs key query value set or list of relevant_docs"
        self.queries_ids = queries_ids
        self.relevant_docs = relevant_docs

        self.mrr_at_k = mrr_at_k
        self.ndcg_at_k = ndcg_at_k
        self.accuracy_at_k = accuracy_at_k
        self.precision_recall_at_k = precision_recall_at_k
        self.map_at_k = map_at_k
    def compute_metrics(self, queries_result_list: List[object]):
        # Init score computation values
        num_hits_at_k = {k: 0 for k in self.accuracy_at_k}
        precisions_at_k = {k: [] for k in self.precision_recall_at_k}
        recall_at_k = {k: [] for k in self.precision_recall_at_k}
        MRR = {k: 0 for k in self.mrr_at_k}
        ndcg = {k: [] for k in self.ndcg_at_k}
        AveP_at_k = {k: [] for k in self.map_at_k}

        # Compute scores on results
        #### elements with hits one hit is a dict : {"corpus_id": corpus_text, "score": score}
        #### corpus_id replace by corpus text
        for query_itr in range(len(queries_result_list)):
            query_id = self.queries_ids[query_itr]

            # Sort scores
            top_hits = sorted(queries_result_list[query_itr], key=lambda x: x['score'], reverse=True)
            query_relevant_docs = self.relevant_docs[query_id]

            # Accuracy@k - We count the result correct, if at least one relevant doc is accross the top-k documents
            for k_val in self.accuracy_at_k:
                for hit in top_hits[0:k_val]:
                    if hit['corpus_id'] in query_relevant_docs:
                        num_hits_at_k[k_val] += 1
                        break

            # Precision and Recall@k
            for k_val in self.precision_recall_at_k:
                num_correct = 0
                for hit in top_hits[0:k_val]:
                    if hit['corpus_id'] in query_relevant_docs:
                        num_correct += 1

                precisions_at_k[k_val].append(num_correct / k_val)
                recall_at_k[k_val].append(num_correct / len(query_relevant_docs))

            # MRR@k
            for k_val in self.mrr_at_k:
                for rank, hit in enumerate(top_hits[0:k_val]):
                    if hit['corpus_id'] in query_relevant_docs:
                        MRR[k_val] += 1.0 / (rank + 1)
                        break

            # NDCG@k
            for k_val in self.ndcg_at_k:
                predicted_relevance = [1 if top_hit['corpus_id'] in query_relevant_docs else 0 for top_hit in top_hits[0:k_val]]
                true_relevances = [1] * len(query_relevant_docs)

                ndcg_value = self.compute_dcg_at_k(predicted_relevance, k_val) / self.compute_dcg_at_k(true_relevances, k_val)
                ndcg[k_val].append(ndcg_value)

            # MAP@k
            for k_val in self.map_at_k:
                num_correct = 0
                sum_precisions = 0

                for rank, hit in enumerate(top_hits[0:k_val]):
                    if hit['corpus_id'] in query_relevant_docs:
                        num_correct += 1
                        sum_precisions += num_correct / (rank + 1)

                avg_precision = sum_precisions / min(k_val, len(query_relevant_docs))
                AveP_at_k[k_val].append(avg_precision)

        # Compute averages
        for k in num_hits_at_k:
            #num_hits_at_k[k] /= len(self.queries)
            num_hits_at_k[k] /= len(queries_result_list)

        for k in precisions_at_k:
            precisions_at_k[k] = np.mean(precisions_at_k[k])

        for k in recall_at_k:
            recall_at_k[k] = np.mean(recall_at_k[k])

        for k in ndcg:
            ndcg[k] = np.mean(ndcg[k])

        for k in MRR:
            #MRR[k] /= len(self.queries)
            MRR[k] /= len(queries_result_list)

        for k in AveP_at_k:
            AveP_at_k[k] = np.mean(AveP_at_k[k])
        return {'accuracy@k': num_hits_at_k, 'precision@k': precisions_at_k, 'recall@k': recall_at_k, 'ndcg@k': ndcg, 'mrr@k': MRR, 'map@k': AveP_at_k}
    @staticmethod
    def compute_dcg_at_k(relevances, k):
        dcg = 0
        for i in range(min(len(relevances), k)):
            dcg += relevances[i] / np.log2(i + 2)  #+2 as we start our idx at 0
        return dcg

def map_dev_samples_to_score_calculator_format(dev_samples):
    if isinstance(dev_samples, dict):
        dev_samples = list(dev_samples.values())
    queries_ids = list(map(lambda x: x["query"] ,dev_samples))
    relevant_docs = dict(map(lambda idx: (dev_samples[idx]["query"], dev_samples[idx]["positive"]), range(len(dev_samples))))
    return ScoreCalculator(queries_ids, relevant_docs)

class ScoreEvaluator:
    """
    This class evaluates a CrossEncoder model for the task of re-ranking.

    Given a query and a list of documents, it computes the score [query, doc_i] for all possible
    documents and sorts them in decreasing order. Then, MRR@10 is compute to measure the quality of the ranking.

    :param samples: Must be a list and each element is of the form: {'query': '', 'positive': [], 'negative': []}. Query is the search query,
     positive is a list of positive (relevant) documents, negative is a list of negative (irrelevant) documents.
    """
    def __init__(self, samples, name: str = '', num_dev_queries = 600):
        self.samples = samples
        self.name = name
        self.num_dev_queries = num_dev_queries

        if isinstance(self.samples, dict):
            self.samples = list(self.samples.values())

        #self.score_calculator_ext = map_dev_samples_to_score_calculator_format(self.samples)

        self.csv_file = "ScoreEvaluator" + ("_" + name if name else '') + "_results.csv"
        self.csv_headers = ["epoch", "steps", "MAP@"]

        self.score_json_file = self.csv_file.replace(".csv", ".json")

    def __call__(self, model, output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
        if epoch != -1:
            if steps == -1:
                out_txt = " after epoch {}:".format(epoch)
            else:
                out_txt = " in epoch {} after {} steps:".format(epoch, steps)
        else:
            out_txt = ":"

        logger.info("ScoreEvaluator: Evaluating the model on " + self.name + " dataset" + out_txt)

        all_scores = []
        num_queries = 0
        num_positives = []
        num_negatives = []

        scores_list = []

        samples = list(self.samples)
        samples_indexes = np.random.permutation(np.arange(len(samples)))
        samples = list(map(lambda idx: samples[idx], samples_indexes[:self.num_dev_queries]))
        #for instance in self.samples:
        for instance in samples:
            query = instance['query']
            positive = list(instance['positive'])
            negative = list(instance['negative'])
            docs = positive + negative
            is_relevant = [True]*len(positive) + [False]*len(negative)

            if len(positive) == 0 or len(negative) == 0:
                continue

            num_queries += 1
            num_positives.append(len(positive))
            num_negatives.append(len(negative))

            model_input = [[query, doc] for doc in docs]
            pred_scores = model.predict(model_input, convert_to_numpy=True, show_progress_bar=False)

            scores_list.extend(list(pred_scores))

            #### elements with hits one hit is a dict : {"corpus_id": corpus_text, "score": score}
            #### corpus_id replace by corpus text
            queries_result_list = list(map(lambda idx: {"corpus_id": docs[idx], "score": pred_scores[idx]}, range(len(docs))))
            score_calculator_ext = map_dev_samples_to_score_calculator_format({0: instance})
            score_dict = score_calculator_ext.compute_metrics([queries_result_list])
            all_scores.append(score_dict["map@k"][100])

        mean_map = np.mean(all_scores)
        logger.info("Queries: {} \t Positives: Min {:.1f}, Mean {:.1f}, Max {:.1f} \t Negatives: Min {:.1f}, Mean {:.1f}, Max {:.1f}".format(num_queries, np.min(num_positives), np.mean(num_positives), np.max(num_positives), np.min(num_negatives), np.mean(num_negatives), np.max(num_negatives)))
        logger.info("MAP@: {:.2f}".format(mean_map*100))

        if output_path is not None:
            csv_path = os.path.join(output_path, self.csv_file)
            output_file_exists = os.path.isfile(csv_path)
            with open(csv_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
                writer = csv.writer(f)
                if not output_file_exists:
                    writer.writerow(self.csv_headers)
                #writer.writerow([epoch, steps, mean_mrr])
                writer.writerow([epoch, steps, mean_map])
            json_path = os.path.join(output_path, self.score_json_file)
            output_file_exists = os.path.isfile(json_path)
            with open(json_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
                writer = csv.writer(f)
                if not output_file_exists:
                    writer.writerow(["epoch", "steps", "score@"])
                writer.writerow([epoch, steps, json.dumps({"scores_list": list(map(float ,scores_list))})])
        #return mean_mrr
        return mean_map

def read_part_file(path, adjust_neg_pos_ration = None):
    json_full_paths = glob(os.path.join(path, "*.json"))
    assert len(json_full_paths) == 4
    req = {}
    for path in json_full_paths:
        val_name = path.split("/")[-1].replace(".json", "")
        with open(path, "r", encoding = "utf-8") as f:
            j_obj = json.load(f)
        if len(j_obj) == 1:
            assert type(j_obj[list(j_obj.keys())[0]]) == type([])
            j_obj = j_obj[list(j_obj.keys())[0]]
        else:
            assert val_name.endswith("_index_dict") and len(val_name.split("_")) == 3
            val_name = "_".join(np.asarray(val_name.split("_"))[[1, 0, 2]].tolist())
            j_obj = dict(map(lambda t2: (t2[1], t2[0]) ,j_obj.items()))
        req[val_name] = j_obj
    if adjust_neg_pos_ration is not None:
        assert type(adjust_neg_pos_ration) == type(0)
        neg_tuple_set = req["neg_tuple_set"]
        pos_tuple_set = req["pos_tuple_set"]
        print("ori pos {} neg {}".format(len(pos_tuple_set), len(neg_tuple_set)))
        if len(pos_tuple_set) < len(neg_tuple_set):
            if len(neg_tuple_set) / len(pos_tuple_set) >= adjust_neg_pos_ration:
                pos_num, neg_num = len(pos_tuple_set), len(pos_tuple_set) * adjust_neg_pos_ration
            else:
                pos_num, neg_num = int(len(neg_tuple_set) / adjust_neg_pos_ration), len(neg_tuple_set)
        else:
            min_size = min(map(len, [neg_tuple_set, pos_tuple_set]))
            snip_size = int(max(1, min_size / 10000))
            snip_num = int(min_size / snip_size)
            pos_num = int(min((snip_num / adjust_neg_pos_ration) * snip_size, min_size))
            neg_num = int(min((snip_num / adjust_neg_pos_ration) * snip_size * adjust_neg_pos_ration, min_size))
        neg_tuple_set = set(list(map(tuple ,neg_tuple_set))[:neg_num])
        pos_tuple_set = set(list(map(tuple ,pos_tuple_set))[:pos_num])
        req["neg_tuple_set"] = neg_tuple_set
        req["pos_tuple_set"] = pos_tuple_set
    return req

def construct_train_samples(json_obj, neg_random = False):
    train_samples = []
    label = 1
    for t2 in json_obj["pos_tuple_set"]:
        q_index, a_index = t2[0], t2[1]
        q, a = json_obj["index_question_dict"][q_index], json_obj["index_answer_dict"][a_index]
        #q = q + "<sep>"
        train_samples.append(InputExample(texts=[q, a], label=label))
    if neg_random:
        neg_len = len(json_obj["index_answer_dict"])
    label = 0
    for t2 in json_obj["neg_tuple_set"]:
        q_index, a_index = t2[0], t2[1]
        if neg_random:
            q, a = json_obj["index_question_dict"][q_index], json_obj["index_answer_dict"][np.random.randint(0, neg_len)]
        else:
            q, a = json_obj["index_question_dict"][q_index], json_obj["index_answer_dict"][a_index]
        #q = q + "<sep>"
        train_samples.append(InputExample(texts=[q, a], label=label))
    train_indexes = np.random.permutation(np.arange(len(train_samples)))
    return list(map(lambda idx: train_samples[idx], train_indexes))

def construct_dev_samples(json_obj ,num_dev_queries = int(2e3), num_max_dev_negatives = 200, neg_random = False):
    dev_samples_list = construct_train_samples(json_obj, neg_random)
    dev_samples_df = pd.DataFrame(list(map(lambda item_:(item_.texts[0], item_.texts[1], item_.label) ,dev_samples_list)), columns = ["q", "a", "l"])
    dev_q_qid_dict = dict(map(lambda t2: (t2[1], t2[0]), enumerate(dev_samples_df["q"].drop_duplicates().tolist())))
    print("dev_samples_df shape {}, dev_q_qid_dict len {}".format(dev_samples_df.shape, len(dev_q_qid_dict)))
    dev_samples = {}
    for q, qid in dev_q_qid_dict.items():
        if qid not in dev_samples and len(dev_samples) < num_dev_queries:
            dev_samples[qid] = {'query': q, 'positive': set(), 'negative': set()}
    for qid in dev_samples.keys():
        qid_relate_df = dev_samples_df[dev_samples_df["q"] == dev_samples[qid]["query"]]
        dev_samples[qid]["positive"] = set(qid_relate_df[qid_relate_df["l"] == 1]["a"].tolist())
        for neg_a in set(qid_relate_df[qid_relate_df["l"] == 0]["a"].tolist()):
            if len(dev_samples[qid]['negative']) < num_max_dev_negatives:
                dev_samples[qid]['negative'].add(neg_a)
    return dev_samples

def merge_dev_samples(dev_samples_list):
    assert len(dev_samples_list) > 1
    query_pos_set_dict = {}
    query_neg_set_dict = {}
    for dev_samples in dev_samples_list:
        for qid, item_ in dev_samples.items():
            # item_ {'query': q, 'positive': set(), 'negative': set()}
            query = item_["query"]
            positive = item_["positive"]
            negative = item_["negative"]
            if query not in query_pos_set_dict:
                query_pos_set_dict[query] = positive
            else:
                for ele in positive:
                    query_pos_set_dict[query].add(ele)
            if query not in query_neg_set_dict:
                query_neg_set_dict[query] = negative
            else:
                for ele in negative:
                    query_neg_set_dict[query].add(ele)
    assert set(query_pos_set_dict.keys()) == set(query_neg_set_dict.keys())
    merge_dev_samples = {}
    for qid, query in enumerate(query_pos_set_dict.keys()):
        merge_dev_samples[qid] =             {'query': query, 'positive': query_pos_set_dict[query], 'negative': query_neg_set_dict[query]}
    return merge_dev_samples

def merge_dev_samples_add_neg(dev_samples_list, add_num = None, after_size = 500):
    assert len(dev_samples_list) > 1
    query_pos_set_dict = {}
    query_neg_set_dict = {}
    all_negs = reduce(lambda a, b: a.union(b) ,map(lambda dev_samples: reduce(lambda a, b: a.union(b) ,map(lambda item_:set(item_["negative"]) ,dev_samples.values())), dev_samples_list))
    assert type(all_negs) == type(set([]))
    assert set(map(type, all_negs)) == set([type("")])
    all_negs = list(all_negs)
    for dev_samples in dev_samples_list:
        for qid, item_ in dev_samples.items():
            # item_ {'query': q, 'positive': set(), 'negative': set()}
            query = item_["query"]
            positive = item_["positive"]
            negative = item_["negative"]
            if query not in query_pos_set_dict:
                query_pos_set_dict[query] = positive
            else:
                for ele in positive:
                    query_pos_set_dict[query].add(ele)
            if query not in query_neg_set_dict:
                query_neg_set_dict[query] = negative
            else:
                for ele in negative:
                    query_neg_set_dict[query].add(ele)
            if add_num is not None:
                assert type(add_num) == type(0)
                all_negs_indexes = np.random.permutation(np.arange(len(all_negs)))
                all_negs_add = list(map(lambda idx: all_negs[idx], all_negs_indexes[:add_num]))
                for ele in all_negs_add:
                    if ele not in positive:
                        query_neg_set_dict[query].add(ele)
    assert set(query_pos_set_dict.keys()) == set(query_neg_set_dict.keys())
    merge_dev_samples = {}
    for qid, query in enumerate(query_pos_set_dict.keys()):
        if len(merge_dev_samples) >= after_size:
            break
        merge_dev_samples[qid] =             {'query': query, 'positive': query_pos_set_dict[query], 'negative': query_neg_set_dict[query]}
    return merge_dev_samples

train_json_obj = read_part_file("train_file_faiss_10", adjust_neg_pos_ration = None)
train_samples = construct_train_samples(train_json_obj, neg_random = True)

valid_json_obj = read_part_file("valid_file_faiss", adjust_neg_pos_ration = None)

dev_samples = construct_dev_samples(valid_json_obj, neg_random = True)

model_name = 'xlm-roberta-base'
model_save_path = 'output/training_ms-marco_cross-encoder-'+model_name.replace("/", "-")+'-'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
model = CrossEncoder_Dict_Eval(model_name, num_labels=1, max_length=512)

train_batch_size = 10
num_epochs = 10
train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)

mrr_sum_evaluator = CERerankingEvaluatorSUM(dev_samples, name='mrr-train-eval')
map_evaluator = ScoreEvaluator(dev_samples, name = "map-train-eval")
dict_evaluator = DictionaryEvaluator([mrr_sum_evaluator, map_evaluator])

warmup_steps = 1000
logging.info("Warmup-steps: {}".format(warmup_steps))

# Train the model
model.fit(train_dataloader=train_dataloader,
          evaluator=dict_evaluator,
          epochs=num_epochs,
          evaluation_steps=5000,
          warmup_steps=warmup_steps,
          output_path=model_save_path,
          use_amp=False)

#Save latest model
model.save(model_save_path+'-latest')

ext: py

max_stars_repo_name: svjack/Sbert-ChineseExample

max_stars_repo_licenses: Unlicense MIT

max_issues_repo_name: svjack/Sbert-ChineseExample

max_issues_repo_licenses: Unlicense MIT

max_forks_repo_name: svjack/Sbert-ChineseExample

max_forks_repo_licenses: Unlicense MIT

reason: import numpy


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'ar'

import cv2
import time
import shutil
import os
import sys
import math
import matplotlib.pyplot as plt
import skimage.io as skio
import skimage.morphology as skmorph
import skimage.transform as sktf
import skimage.exposure as skexp
import numpy as np
import keras
from keras.layers import Conv2D, UpSampling2D, \
    Flatten, Activation, Reshape, MaxPooling2D, Input, merge
from keras.models import Model
import keras.losses
import keras.callbacks as kall
import pandas as pd
import keras.optimizers as kopt

from keras.preprocessing.image import ImageDataGenerator
from keras.utils.vis_utils import plot_model as kplot
from keras.utils import np_utils
from keras.utils.vis_utils import plot_model

#####################################################
def buildModelFCNNCLS_UpSampling2D_V3(inpShape=(256, 256, 3),
                                      numCls=4,
                                      numConv=2, kernelSize=3, numFlt=8,
                                      isUNetStyle=True,
                                      unetStartLayer=1,
                                      ppad='same', numSubsampling=5, isDebug=False):
    dataInput = Input(shape=inpShape)
    fsiz = (kernelSize, kernelSize)
    psiz = (2, 2)
    x = dataInput
    # -------- Encoder --------
    lstMaxPools = []
    for cc in range(numSubsampling):
        for ii in range(numConv):
            x = Conv2D(filters=numFlt * (2**cc), kernel_size=fsiz, padding=ppad, activation='relu')(x)
        lstMaxPools.append(x)
        x = MaxPooling2D(pool_size=psiz)(x)
    # -------- Decoder --------
    for cc in range(numSubsampling):
        for ii in range(numConv):
            x = Conv2D(filters=numFlt * (2 ** (numSubsampling - 1 -cc)), kernel_size=fsiz, padding=ppad, activation='relu')(x)
        x = UpSampling2D(size=psiz)(x)
        if isUNetStyle:
            if cc<(numSubsampling-unetStartLayer):
                x = keras.layers.concatenate([x, lstMaxPools[-1 - cc]], axis=-1)
    #
    # 1x1 Convolution: emulation of Dense layer
    x = Conv2D(filters=numCls, kernel_size=(1,1), padding='valid', activation='linear')(x)
    x = Reshape([-1, numCls])(x)
    x = Activation('softmax')(x)
    retModel = Model(dataInput, x)
    if isDebug:
        retModel.summary()
        fimg_model = 'model_graph_FCNN_UpSampling2D_V3.png'
        kplot(retModel, fimg_model, show_shapes=True)
        plt.imshow(skio.imread(fimg_model))
        plt.show()
    return retModel

#####################################################
def preproc_image(pimg):
    ndim = pimg.ndim
    # trnd = np.random.randint(4)
    trnd = 1
    timg = pimg[:, :, :3]
    ret = pimg.copy()
    if trnd == 0:
        timg = skexp.equalize_hist(timg.astype(np.uint8)).astype(np.float32) * 255.
    elif trnd == 1:
        # vrnd = 1.0 + 0.2 * ( np.random.rand() - 0.5)
        vrnd = getRandomInRange((0.3, 2.5))
        timg = (255.*skexp.adjust_gamma(timg.astype(np.float32)/255., vrnd)).astype(np.uint8)
    elif trnd > 1:
        rndVals = 2.0 * np.random.rand(ndim,2) - 1.0
        rndVals[:, 0] *= 30
        rndVals[:, 1] = 1.0 + 0.2 * rndVals[:, 1]
        for ii in range(ndim):
            timg[:,:,ii] = rndVals[ii,0] + rndVals[ii,1] * timg[:,:,ii]
    timg[timg < 0] = 0
    timg[timg > 255] = 255
    ret[:, :,:3] = timg.copy()
    return ret

#####################################################
def readDataImagesCls(pidx, wdir=None, maxNum=None):
    if wdir is None:
        wdir = os.path.dirname(pidx)
    tdata = pd.read_csv(pidx)
    if maxNum is not None:
        numData = len(tdata)
        if maxNum > numData:
            maxNum = numData
        tdata = tdata[:maxNum]
    #
    dataCls = tdata['type'].as_matrix()
    lstpath = tdata['path'].as_matrix()
    lstpath = [os.path.join(wdir, xx) for xx in lstpath]
    dataPaths = lstpath
    numPath = len(lstpath)
    dataX = None
    print (':: read images into memory...')
    for ipath, path in enumerate(lstpath):
        timg = skio.imread(path)
        if dataX is None:
            dataX = np.zeros([numPath] + list(timg.shape), dtype=np.uint8)
        if (ipath % 20) == 0:
            print ('\t[{0}/{1}]'.format(ipath, numPath))
        dataX[ipath] = timg
    return dataX, dataCls, dataPaths

#####################################################
def calcDistArr2Point(parr2d, pp2d):
    sizArr = parr2d.shape[0]
    ret = np.linalg.norm(parr2d - np.tile(pp2d, (sizArr,1)), axis=1)
    return ret

def buildImageWithRotScaleAroundCenter(pimg, pcnt, pangDec, pscale, pcropSize, isDebug=False):
    # (1) precalc parameters
    angRad = (np.pi / 180.) * pangDec
    cosa = np.cos(angRad)
    sina = np.sin(angRad)
    # (2) prepare separate affine transformation matrices
    matShiftB = np.array([[1., 0., -pcnt[0]], [0., 1., -pcnt[1]], [0., 0., 1.]])
    matRot = np.array([[cosa, sina, 0.], [-sina, cosa, 0.], [0., 0., 1.]])
    matShiftF = np.array([[1., 0., +pcnt[0]], [0., 1., +pcnt[1]], [0., 0., 1.]])
    matScale = np.array([[pscale, 0., 0.], [0., pscale, 0.], [0., 0., 1.]])
    matShiftCrop = np.array([[1., 0., pcropSize[0] / 2.], [0., 1., pcropSize[1] / 2.], [0., 0., 1.]])
    # matTotal_OCV = matShiftF.dot(matRot.dot(matScale.dot(matShiftB)))
    # (3) build total-matrix
    matTotal = matShiftCrop.dot(matRot.dot(matScale.dot(matShiftB)))
    if isDebug:
        print ('(1) mat-shift-backward = \n{0}'.format(matShiftB))
        print ('(2) mat-scale = \n{0}'.format(matScale))
        print ('(3) mat-rot = \n{0}'.format(matRot))
        print ('(4) mat-shift-forward = \n{0}'.format(matShiftF))
        print ('(5) mat-shift-crop = \n{0}'.format(matShiftCrop))
        print ('---\n(*) mat-total = \n{0}'.format(matTotal))
    # (4) warp image with total affine-transform
    imgRet = cv2.warpAffine(pimg, matTotal[:2, :], pcropSize, flags=cv2.INTER_NEAREST)
    return imgRet

def prepareCervixAndChannelInfo(pimg, pRelChnSize = 0.7, isDebug = False):
    # (1) prepare masks
    tmsk = pimg[:, :, 3]
    timg = pimg[:, :, :3]
    tmsk_chn = (tmsk == 128)
    tmsk_crv = (tmsk > 100)
    # rc - mean first-idx -> row, second-idx -> column, xy - mean first-idx -> column, second-idx -> row :)
    # (2) find channel cover-circle and center of this corcle
    rc_pts_channel = np.array(np.where(tmsk_chn)).transpose()
    (rc_channel_cnt, r_channel) = cv2.minEnclosingCircle(rc_pts_channel)
    dist_chn2cnt = calcDistArr2Point(rc_pts_channel, rc_channel_cnt)
    r_channel_good = rc_pts_channel[dist_chn2cnt < pRelChnSize * r_channel, :]
    #FIXME: Fill holes before this step!!!
    # (2) prepare cervix contour
    contour_crv = tmsk_crv & (~skmorph.erosion(tmsk_crv, skmorph.disk(1)))
    rc_crvContour = np.array(np.where(contour_crv)).transpose()
    dist_contour2cnt = calcDistArr2Point(rc_crvContour, rc_channel_cnt)
    r_cervix_min = np.min(dist_contour2cnt)
    r_cervix_max = np.max(dist_contour2cnt)
    # rcCrvRminArg = np.argmin(rcRContour)
    if r_cervix_min<r_channel:
        r_cervix_min = r_channel
    ret = {
        'r_crv_min': r_cervix_min,
        'r_crv_max': r_cervix_max,
        'r_chn': r_channel,
        'r_chn_good': pRelChnSize * r_channel,
        'cnt_chn': rc_channel_cnt,
        'rc_chn': r_channel_good.copy()
    }
    if isDebug:
        retSize = 256
        newScale =  float(retSize)/(2.*r_cervix_min + 2.)
        xy_channel_cnt = rc_channel_cnt[::-1]
        timg_crop = buildImageWithRotScaleAroundCenter(timg, xy_channel_cnt, 45., newScale, (retSize, retSize), isDebug=False)
        #
        plt.subplot(2, 2, 1)
        plt.imshow(tmsk_chn)
        plt.gcf().gca().add_artist(plt.Circle(rc_channel_cnt[::-1], r_channel, edgecolor='r', fill=False))
        plt.gcf().gca().add_artist(plt.Circle(rc_channel_cnt[::-1], r_cervix_min, edgecolor='g', fill=False))
        plt.plot(r_channel_good[:, 1], r_channel_good[:, 0], 'y.')
        plt.subplot(2, 2, 2)
        plt.imshow(tmsk_crv)
        plt.gcf().gca().add_artist(plt.Circle(rc_channel_cnt[::-1], r_channel, edgecolor='r', fill=False))
        plt.gcf().gca().add_artist(plt.Circle(rc_channel_cnt[::-1], r_cervix_min, edgecolor='g', fill=False))
        plt.subplot(2, 2, 3)
        plt.imshow(timg)
        plt.gcf().gca().add_artist(plt.Circle(rc_channel_cnt[::-1], r_channel, edgecolor='r', fill=False))
        plt.gcf().gca().add_artist(plt.Circle(rc_channel_cnt[::-1], r_cervix_min, edgecolor='g', fill=False))
        plt.subplot(2, 2, 4)
        plt.imshow(timg_crop)
        plt.show()
    return ret

def buildImgInfoList(dataImg):
    numImg = dataImg.shape[0]
    print (":: Prepare image info ({0})".format(dataImg.shape))
    ret = []
    for ii in range(numImg):
        timg = dataImg[ii]
        tinfo = prepareCervixAndChannelInfo(timg)
        ret.append(tinfo)
        if (ii%10)==0:
            print ('\t[{0}/{1}]'.format(ii, numImg))
    return ret

#####################################################
def getRandomInRange(vrange, pnum=None):
    vmin,vmax = vrange
    if pnum is None:
        trnd = np.random.rand()
    else:
        trnd = np.random.rand(pnum)
    ret = vmin + (vmax-vmin)*trnd
    return ret

#####################################################
def train_generator_V1(dataImg, dataCls, dataInfo, numCls=4, batchSize=64, imsize = 256, isRandomize=True, angleRange=(-24., +24.), isDebug=False):
    numImg   = dataImg.shape[0]
    shapeMsk = dataImg.shape[1:3]
    # #Cls is equal number of channel types + background
    cropSize = (imsize, imsize)
    while True:
        rndIdx = np.random.randint(0, numImg, batchSize)
        dataX = np.zeros((batchSize, imsize, imsize, 3))
        dataY = np.zeros((batchSize, imsize * imsize, numCls))
        rndIsFlip = np.random.rand(batchSize)>0.5
        rndShiftMean = 0.2 * getRandomInRange((-1., 1.0), pnum=batchSize)
        rndShiftStd = 1.0 + 0.2 * getRandomInRange((-1.0, 1.0), pnum=batchSize)
        for ii, idx in enumerate(rndIdx):
            timg4 = dataImg[idx]
            tinf = dataInfo[idx]
            tcls = dataCls[idx]
            # (1) get random point around channel
            PTS_chn_rc = tinf['rc_chn']
            rndChnPos = np.random.randint(PTS_chn_rc.shape[0])
            Pxy = PTS_chn_rc[rndChnPos][::-1]
            #
            Rchn0 = tinf['r_chn']
            Rcrv1 = tinf['r_crv_min']
            Rcrv2 = tinf['r_crv_max']
            #
            Rmin = Rchn0 * 3
            if Rmin>Rcrv1:
                Rmin = Rcrv1
            Rmax = Rcrv2
            if Rcrv2>2*Rcrv1:
                Rmax = 2*Rcrv1
            Rrnd = getRandomInRange((Rmin, Rmax))
            Arnd = getRandomInRange(angleRange)
            cropScale = float(imsize)/Rrnd
            timgCrop = buildImageWithRotScaleAroundCenter(timg4, Pxy, Arnd, cropScale, cropSize, isDebug=False)
            #
            if rndIsFlip[ii]:
                timgCrop = np.fliplr(timgCrop)
            #
            tmsk0 = (timgCrop[:,:,3].reshape(-1)==128).astype(np.uint8)
            tmsk0[tmsk0>0] = tcls
            tmskCls = np_utils.to_categorical(tmsk0, numCls)
            #
            timg3 = timgCrop[:, :, :3]
            #
            # if isRandomize:
            #     timg3 = preproc_image(timg3)
            timg3 = timg3.astype(np.float32) / 127.5 - 1.0
            if isRandomize:
                timg3 -= rndShiftMean[ii]
                timg3 *= rndShiftStd[ii]
            #
            dataX[ii] = timg3
            dataY[ii] = tmskCls
            if isDebug:
                plt.subplot(2, 2, 1)
                plt.imshow(timg4[:,:,:3])
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rmin, edgecolor='r', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rrnd, edgecolor='y', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rmax, edgecolor='g', fill=False))
                plt.subplot(2, 2, 2)
                plt.imshow(timg4[:,:,3])
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rmin, edgecolor='r', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rrnd, edgecolor='y', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rmax, edgecolor='g', fill=False))
                plt.subplot(2, 2, 3)
                plt.imshow(timgCrop[:, :,:3])
                plt.subplot(2, 2, 4)
                plt.imshow(timgCrop[:, :, 3])
                plt.show()
        yield (dataX, dataY)

def train_generator_V2(dataImg, dataCls, dataInfo=None, numCls=4, batchSize=64, imsize=256,
                       isRandomize=True, angleRange=(-36., +36.), scaleRange=(0.9, 1.2),
                       isDebug=False):
    numImg = dataImg.shape[0]
    shapeMsk = dataImg.shape[1:3]
    # #Cls is equal number of channel types + background
    cropSize = (imsize, imsize)
    while True:
        rndIdx = np.random.randint(0, numImg, batchSize)
        dataX = np.zeros((batchSize, imsize, imsize, 3))
        dataY = np.zeros((batchSize, imsize * imsize, numCls))
        rndIsFlip = np.random.rand(batchSize) > 0.5
        # rndShiftMean = 0.2 * getRandomInRange((-1., 1.0), pnum=batchSize)
        # rndShiftStd = 1.0 + 0.2 * getRandomInRange((-1.0, 1.0), pnum=batchSize)
        rndAngPos = np.random.rand(batchSize) * 2. * math.pi
        rndCosPos = np.cos(rndAngPos)
        rndSinPos = np.sin(rndAngPos)
        rndRadPos = np.random.rand(batchSize)
        rndScale = getRandomInRange(scaleRange, pnum=batchSize)
        rndAngle = getRandomInRange(angleRange, pnum=batchSize)
        for ii, idx in enumerate(rndIdx):
            timg4 = dataImg[idx]
            tinf = dataInfo[idx]
            tcls = dataCls[idx]
            # (1) get random point around channel
            PChannelXY = tinf['rc_chn'][::-1].mean(axis=0)
            #
            RChannel = tinf['r_chn']
            RCervix = 0.5*(tinf['r_crv_max'] + tinf['r_crv_min'])
            Rrnd = RChannel
            if RChannel > (0.5 * RCervix):
                Rrnd = 0.5 * RCervix
            trad = rndRadPos[ii] * Rrnd
            x00 = int(PChannelXY[0] + trad * rndCosPos[ii])
            y00 = int(PChannelXY[1] + trad * rndSinPos[ii])
            Pxy = (x00, y00)
            timgCrop = buildImageWithRotScaleAroundCenter(timg4, Pxy, rndAngle[ii], rndScale[ii], cropSize, isDebug=False)
            #
            if rndIsFlip[ii]:
                timgCrop = np.fliplr(timgCrop)
            #
            tmsk0 = (timgCrop[:, :, 3].reshape(-1) == 128).astype(np.uint8)
            tmsk0[tmsk0 > 0] = tcls
            tmskCls = np_utils.to_categorical(tmsk0, numCls)
            #
            timg3 = timgCrop[:, :, :3]
            #
            if isRandomize:
                timg3 = preproc_image(timg3)
            timg3 = timg3.astype(np.float32) / 127.5 - 1.0
            # if isRandomize:
            #     timg3 -= rndShiftMean[ii]
            #     timg3 *= rndShiftStd[ii]
            #
            dataX[ii] = timg3
            dataY[ii] = tmskCls
            if isDebug:
                plt.subplot(2, 2, 1)
                plt.imshow(timg4[:, :, :3])
                plt.gcf().gca().add_artist(plt.Circle(Pxy, RChannel, edgecolor='r', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rrnd, edgecolor='y', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, RCervix, edgecolor='g', fill=False))
                plt.subplot(2, 2, 2)
                plt.imshow(timg4[:, :, 3])
                plt.gcf().gca().add_artist(plt.Circle(Pxy, RChannel, edgecolor='r', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, Rrnd, edgecolor='y', fill=False))
                plt.gcf().gca().add_artist(plt.Circle(Pxy, RCervix, edgecolor='g', fill=False))
                plt.subplot(2, 2, 3)
                plt.imshow(timgCrop[:, :, :3])
                plt.subplot(2, 2, 4)
                plt.imshow(timgCrop[:, :, 3])
                plt.show()
            # print ('-')
        yield (dataX, dataY)

#####################################################
if __name__ == '__main__':
    imgSize = 256
    numCls = 4
    # (1) Setup Tran/Validation data
    # fidxTrn = '/home/ar/@Kaggle/01_Intel_&_MobileODT_Cervical_Cancer_Screening/data/train-original-1024x1024-bordered/idx.txt-train.txt'
    # fidxVal = '/home/ar/@Kaggle/01_Intel_&_MobileODT_Cervical_Cancer_Screening/data/train-original-1024x1024-bordered/idx.txt-val.txt'
    fidxTrn = '/home/ar/@Kaggle/01_Intel_&_MobileODT_Cervical_Cancer_Screening/data/train-original-512x512-bordered/idx.txt-train.txt'
    fidxVal = '/home/ar/@Kaggle/01_Intel_&_MobileODT_Cervical_Cancer_Screening/data/train-original-512x512-bordered/idx.txt-val.txt'
    wdir = os.path.dirname(fidxTrn)
    # q = next(train_generator(trnX, trnCls, trnInfo, isDebug=False))
    #
    # (2) Input/Output models
    pathModelPrefix = '{0}/model_fcncls_channel_V2'.format(wdir)
    pathModelValLoss = '{0}_valLoss_v1.h5'.format(pathModelPrefix)
    pathModelValAcc = '{0}_valAcc_v1.h5'.format(pathModelPrefix)
    pathModelLatest = '{0}_Latest_v1.h5'.format(pathModelPrefix)
    pathLog = '%s-log.csv' % pathModelValLoss
    # (3) Visualise model (for test)
    #
    # (4) Continue training from checkpoint Model (if exists)
    pathModelRestart = pathModelValLoss
    if not os.path.isfile(pathModelRestart):
        model = buildModelFCNNCLS_UpSampling2D_V3(inpShape=(imgSize, imgSize, 3),
                                                  numCls=numCls)
        model.compile(optimizer='adam',
                      loss='categorical_crossentropy',
                      metrics=['accuracy'])
    else:
        pref = time.strftime('%Y.%m.%d-%H.%M.%S')
        pathModelValBk = '%s-%s.bk' % (pathModelValLoss, pref)
        pathModelValAccBk = '%s-%s.bk' % (pathModelValAcc, pref)
        pathModelLatestBk = '%s-%s.bk' % (pathModelLatest, pref)
        shutil.copy(pathModelValLoss, pathModelValBk)
        shutil.copy(pathModelValAcc, pathModelValAccBk)
        # shutil.copy(pathModelLatest, pathModelLatestBk)
        model = keras.models.load_model(pathModelRestart)
        #
        # model.compile(optimizer=kopt.Adam(lr=0.00004), loss='categorical_crossentropy', metrics=['accuracy'])
    # (5) Preload data
    trnX, trnCls, _ = readDataImagesCls(fidxTrn, maxNum=None)
    valX, valCls, _ = readDataImagesCls(fidxVal, maxNum=None)
    trnInfo = buildImgInfoList(trnX)
    valInfo = buildImgInfoList(valX)
    numTrn = trnX.shape[0]
    numVal = valX.shape[0]
    #
    batchSize = 32
    numEpochs = 1000
    numIterPerEpochTrn = numTrn / batchSize
    if numIterPerEpochTrn<1:
        numIterPerEpochTrn = 1
    numIterPerEpochVal = numVal / batchSize
    if numIterPerEpochVal < 1:
        numIterPerEpochVal = 1
    # dataTrn = dataVal
    # valXg, valYg = next(train_generator_V2(dataImg=valX,
    #                                        dataCls=valCls,
    #                                        dataInfo=valInfo,
    #                                        imsize=imgSize,
    #                                        batchSize=1024,
    #                                        isRandomize=False, isDebug=False))
    valGenerator = train_generator_V2(
                    dataImg=trnX, dataCls=trnCls, dataInfo=trnInfo,
                    imsize=imgSize, batchSize=batchSize, isRandomize=False)
    #
    model.fit_generator(
        generator=train_generator_V2(
                    dataImg=trnX, dataCls=trnCls, dataInfo=trnInfo,
                    imsize=imgSize, batchSize=batchSize, isRandomize=True),
        steps_per_epoch=numIterPerEpochTrn,
        epochs=numEpochs,
        # validation_data=(valXg, valYg),
        validation_data=valGenerator,
        validation_steps=numIterPerEpochVal,
        callbacks=[
            kall.ModelCheckpoint(pathModelValLoss, verbose=True, save_best_only=True, monitor='val_loss'),
            kall.ModelCheckpoint(pathModelValAcc, verbose=True, save_best_only=True, monitor='val_acc'),
            # kall.ModelCheckpoint(pathModelLatest, verbose=True, save_best_only=False),
            kall.CSVLogger(pathLog, append=True)
        ])


text:
# -*- coding: UTF-8 -*-
# Copyright (c) 2021 Baidu, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Pauli Measurement Circuit
"""

from typing import Dict, List
from copy import deepcopy
import itertools
import numpy as np
from QCompute.QPlatform.QEnv import QEnv
from QCompute.QPlatform.QRegPool import QRegPool
from QCompute.QPlatform.QOperation.RotationGate import RX, RY
from QCompute.QPlatform.QOperation.FixedGate import H, S, CX
from QCompute.QPlatform.QOperation.Measure import MeasureZ
from qapp.utils import grouping_hamiltonian
from .basic_circuit import BasicCircuit

class PauliMeasurementCircuit(BasicCircuit):
    """ Pauli Measurement Circuit class
    """
    def __init__(self, num: int, pauli_terms: str):
        """The constructor of the PauliMeasurementCircuit class

        :param num: Number of qubits
        :param pauli_terms: Pauli terms to be measured
        """
        super().__init__(num)
        self._pauli_terms = deepcopy(pauli_terms)

    def add_circuit(self, q: QRegPool, pauli_str: str):
        """Adds the pauli measurement circuit to the register

        :param q: Quantum register to which this circuit is added
        :param pauli_str: Pauli string to be measured
        """
        for i in range(self._num):
            # Set up Pauli measurement circuit
            if pauli_str[i] == 'x':
                RY(-np.pi / 2)(q[i])
            elif pauli_str[i] == 'y':
                RX(np.pi / 2)(q[i])
        # Measure all qubits
        MeasureZ(q, list(range(len(q))))

    def get_expectation(self, preceding_circuits: List[BasicCircuit], shots: int, backend: str) -> float:
        """Computes the expectation value of the Pauli terms

        :param preceding_circuit: Circuit precedes the measurement circuit
        :param shots: Number of measurement shots
        :param backend: Backend to be used in this task
        :return: Expectation value of the Pauli terms
        """
        expectation = 0
        for coeff, pauli_str in self._pauli_terms:
            # Return coeff if all Pauli operators are I
            if pauli_str.lower().count('i') == len(pauli_str):
                expectation += coeff
                continue
            active_qubits = [i for i, c in enumerate(pauli_str) if c != 'i']
            env = QEnv()
            env.backend(backend)
            q = env.Q.createList(self._num)
            # Add circuit
            for circuit in preceding_circuits:
                circuit.add_circuit(q)
            self.add_circuit(q, pauli_str.lower())
            # Submit job
            counts = env.commit(shots, fetchMeasure=True)['counts']
            # Expectation
            filtered_counts = [(counts[key], [key[-i - 1] for i in active_qubits]) for key in counts]
            expecval = sum([((-1) ** key.count('1')) * val / shots for val, key in filtered_counts])
            expectation += coeff * expecval

        return expectation

class PauliMeasurementCircuitWithAncilla(BasicCircuit):
    """Pauli Measurement Circuit with Ancilla class
    """
    def __init__(self, num: int, pauli_terms: str):
        """The constructor of the PauliMeasurementCircuitWithAncilla class

        :param num: Number of qubits
        :param pauli_terms: Pauli terms to be measured
        """
        super().__init__(num)
        self._pauli_terms = pauli_terms

    def add_circuit(self, q: QRegPool, pauli_str: str):
        """Adds the pauli measurement circuit to the register

        :param q: Quantum register to which this circuit is added
        :param pauli_str: Pauli string to be measured
        """
        for i in range(self._num):
            # Set up Pauli measurement circuit
            if pauli_str[i] == 'x':
                H(q[i])
                CX(q[i], q[-1])
            elif pauli_str[i] == 'y':
                S(q[i])
                H(q[i])
                CX(q[i], q[-1])
            elif pauli_str[i] == 'z':
                CX(q[i], q[-1])
        # Measure the ancilla qubit
        MeasureZ([q[-1]], [0])

    def get_expectation(self, preceding_circuits: List[BasicCircuit], shots: int, backend: str) -> float:
        """Computes the expectation value of the Pauli terms

        :param preceding_circuit: Circuit precedes the measurement circuit
        :param shots: Number of measurement shots
        :param backend: Backend to be used in this task
        :return: Expectation value of the Pauli terms
        """
        expectation = 0
        for coeff, pauli_str in self._pauli_terms:
            # Return coeff if all Pauli operators are I
            if pauli_str.lower().count('i') == len(pauli_str):
                expectation += coeff
                continue
            env = QEnv()
            env.backend(backend)
            q = env.Q.createList(self._num + 1)
            # Add circuit
            for circuit in preceding_circuits:
                circuit.add_circuit(q[:self._num])
            self.add_circuit(q, pauli_str.lower())
            # Submit job
            counts = env.commit(shots, fetchMeasure=True)['counts']
            # Expectation
            expecval = (counts.get('0', 0) - counts.get('1', 0)) / shots
            expectation += coeff * expecval

        return expectation

class SimultaneousPauliMeasurementCircuit(BasicCircuit):
    """Simultaneous Pauli Measurement Circuit for Qubitwise Commute Pauli Terms
    """
    def __init__(self, num: int, pauli_terms: List):
        """The constructor of the SimultaneousPauliMeasurementCircuit class

        :param num: Number of qubits
        :param pauli_terms: Pauli terms to be measured
        """
        super().__init__(num)
        self._pauli_terms = pauli_terms

    def add_circuit(self, q: 'QRegPool', clique: List):
        """Adds the simultaneous pauli measurement circuit to the register

        :param q: Quantum register to which this circuit is added
        :param clique: Clique of Pauli terms to be measured together
        """
        for index in range(self._num):
            # Set up Pauli measurement circuit
            term = [pauli[index] for _, pauli in clique]
            if 'x' in term:
                RY(-np.pi / 2)(q[index])
            elif 'y' in term:
                RX(np.pi / 2)(q[index])
        # Measure all qubits
        MeasureZ(q, range(self._num))

    def _single_clique_expectation(self, clique: List, counts: Dict, shots: int) -> float:
        """Computes the expectation value of the target Pauli clique
        """
        # Reformulate the measurement counts in the form of a probability list
        basis = [''.join(x) for x in itertools.product('01', repeat=self._num)]
        prob = [counts.get(key, 0) / shots for key in basis]
        # Calculate the expectation value of each term in the pauli_clique according to the same measured results
        expecval = 0
        for coeff, pauli_str in clique:
            first = True
            for operator in pauli_str:
                if not first:
                    eigenvalues = np.kron(np.array([1.0, 1.0 if operator == 'i' else -1.0]), eigenvalues)
                else:
                    eigenvalues = np.array([1.0, 1.0 if operator == 'i' else -1.0])
                    first = False
            expecval_index = np.dot(eigenvalues, prob)
            expecval += coeff * expecval_index

        return expecval

    def get_expectation(self, preceding_circuits: List[BasicCircuit], shots: int, backend: str) -> float:
        """Computes the expectation value of the Pauli terms

        :param preceding_circuit: Circuit precedes the measurement circuit
        :param shots: Number of measurement shots
        :param backend: Backend to be used in this task
        :return: Expectation value of the Pauli terms
        """
        # Generate Pauli cliques for the hamiltonian graph
        cliques = grouping_hamiltonian(self._pauli_terms)
        # Calculate each
        expectation = 0
        for pauli_clique in cliques:
            env = QEnv()
            env.backend(backend)
            q = env.Q.createList(self._num)
            # Add circuit
            for circuit in preceding_circuits:
                circuit.add_circuit(q)
            self.add_circuit(q, pauli_clique)
            # Submit job
            counts = env.commit(shots, fetchMeasure=True)['counts']
            # Expectation
            result = self._single_clique_expectation(pauli_clique, counts, shots)
            expectation += result

        return expectation

max_stars_repo_name: baidu/QCompute

max_issues_repo_name: baidu/QCompute

max_forks_repo_name: baidu/QCompute


text:
from collections import OrderedDict
from torch.nn.functional import one_hot
from torch.utils.data import DataLoader
import numpy as np
import torch

def words_token(sent,word_to_ix):
    list_ = np.zeros(shape=(len(sent),1)).astype('float64')
    for i,data in enumerate(sent):
        list_[i] = word_to_ix[data]
    return list_

# use one of the label_token for specific classification task

def label_token(label): # for 2 way classification
    mapping = {'true': 1, 'mostly-true': 1, 'half-true': 1, 'barely-true': 0, 'false': 0, 'pants-fire': 0}
    return [mapping[label]]

# def label_token(label): # for 6 way classification
#     mapping = {'true': 5, 'mostly-true': 4, 'half-true': 3, 'barely-true': 2, 'false': 1, 'pants-fire': 0}
#     return [mapping[label]]

def into_token(direct,word_to_ix):
    train_= []
    for i in direct:
        word_ = words_token(i[0],word_to_ix)
        label_ = label_token(i[1])
        train_.append([word_,label_])

    return np.array(train_)

def word_to_ix_(data):
    word_to_ix = OrderedDict()
    for sent in data :
        for word in sent:
            if word not in word_to_ix:
                word_to_ix[word] = len(word_to_ix)
    return word_to_ix

def label_to_ix_(data):
    label_to_ix = OrderedDict()
    for sent in data:
        for label in sent:
            if label not in label_to_ix:
                label_to_ix[label] = len(label_to_ix)
    return label_to_ix

def make_target(label,label_to_ix):
    return torch.LongTensor([label_to_ix[label]])

def sent_words(data,label = False):
    lis = []
    if label == False:
        for i in range(len(data)):
            lis.append(data['statement'][i].split())
    else:
        for i in range(len(data)):
            lis.append(data['label'][i].split())
    return np.array(lis)

def one_hot_(sent):
    words = torch.tensor([word_to_ix[w] for w in sent ], dtype=torch.long)
    print(words)
    one_hot_encoding = one_hot(words)
    return one_hot_encoding


text:
from typing import Any, Dict, List, Optional

import numpy as np
import pandas as pd
from pandas.testing import assert_frame_equal
import pytest

import altair_transform

@pytest.fixture
def data() -> pd.DataFrame:
    return pd.DataFrame({"x": [1, 2, 2], "y1": ["A", "B", "C"], "y2": ["D", "E", "F"]})

@pytest.mark.parametrize("as_", (None, ["name", "val"]))
def test_fold_transform(data, as_: Optional[List[str]]):
    if as_ is None:
        out = altair_transform.apply(data, {"fold": ["y1", "y2"]})
        as_ = ["key", "value"]
    else:
        out = altair_transform.apply(data, {"fold": ["y1", "y2"], "as": as_})

    expected = pd.DataFrame(
        {
            "x": np.repeat(data["x"], 2),
            as_[0]: 3 * ["y1", "y2"],
            as_[1]: np.ravel((data["y1"], data["y2"]), "F"),
            "y1": np.repeat(data["y1"], 2),
            "y2": np.repeat(data["y2"], 2),
        }
    ).reset_index(drop=True)
    assert_frame_equal(out, expected)

@pytest.mark.parametrize("fold", [["y1"], ["y1", "y2"]])
@pytest.mark.parametrize("as_", [None, ["name", "val"]])
def test_fold_against_js(
    driver, data: pd.DataFrame, fold: List[str], as_: Optional[str]
) -> None:
    transform: Dict[str, Any] = {"fold": fold}
    if as_ is not None:
        transform["as"] = as_

    got = altair_transform.apply(data, transform)
    want = driver.apply(data, transform)

    assert_frame_equal(
        got[sorted(got.columns)],
        want[sorted(want.columns)],
        check_dtype=False,
        check_index_type=False,
        check_less_precise=True,
    )

max_stars_repo_name: altair-viz/altair-transform

max_issues_repo_name: altair-viz/altair-transform

max_forks_repo_name: altair-viz/altair-transform


text:
import librosa as lb
from librosa.display import specshow
import python_speech_features
import matplotlib.pyplot as plt
from scipy.signal.windows import hann
import os
from os import listdir
from os.path import isfile, join
import numpy as np

import pandas as pd

def is_wav(filename):
    '''
        Checks if files are .wav files
        Utility tool in converting wav to png files
    '''
    return filename.split('.')[-1] == 'wav'
#create images using librosa spectogram
def convert_to_spec_image(file_loc, filename, category, is_train=False, verbose=False):
    '''
        Converts audio file to spec image
        Input file includes path
        Saves the file to a png image in the save_directory
    '''
    train_ = 'train/'
    val_ = 'val/'

    loc = file_loc + train_ + category + '/' + filename
    if is_train == False:
        loc = file_loc + val_ + category + '/' + filename

    if verbose == True:
        print('reading and converting ' + filename + '...')

    y, sr = lb.load(loc)

        #Convert audio to mfcc
    n_mfcc = 13
    n_mels = 40
    n_fft = 512
    hop_length = 160 #smaples
    fmin = 0
    fmax = None
    mfcc_speech = python_speech_features.mfcc(signal=y, samplerate=sr, winlen=n_fft / sr, winstep=hop_length / sr,
                                          numcep=n_mfcc, nfilt=n_mels, nfft=n_fft, lowfreq=fmin, highfreq=fmax,
                                          preemph=0.0, ceplifter=0, appendEnergy=False, winfunc=hann)
    mfcc_speech= np.swapaxes(mfcc_speech, 0 ,1)
    specshow(mfcc_speech,sr=sr)
    plt.axis('off')
    #plt.show()
    #breakpoint()
    save_directory = 'output/'
    filename_img = filename.split('.wav')[0]

    save_loc = save_directory + train_ + category + '/' + filename_img + '.png'
    if is_train == False:
        save_loc = save_directory + val_ + category + '/' + filename_img + '.png'

    plt.savefig(save_loc, bbox_inches="tight")
    #breakpoint()
    if verbose == True:
        print(filename + ' converted!')

    plt.close()

if __name__ == '__main__':
    files_loc = './cnn-part-2-split-to-train-and-test/output/'
    diagnosis_csv = './respiratory-sound-database/respiratory_sound_database/Respiratory_Sound_Database/patient_diagnosis.csv'
    diagnosis = pd.read_csv(diagnosis_csv, names=['pId', 'diagnosis'])
    categories = diagnosis['diagnosis'].unique()
    if os.path.exists('./output'):
        pass
    else:
        os.makedirs('output')
        os.makedirs('output/train')
        os.makedirs('output/val')
        for cat in categories:
            os.makedirs('output/train/' + cat)
            os.makedirs('output/val/' + cat)

    split = ['train', 'val']
    for s in split:
        for cat in categories:
            #print('-' * 100)
            #print('working on ' + cat + '...')
            #print('-' * 100)

            files = [f for f in listdir(files_loc + s + '/' + cat + '/') if isfile(join(files_loc + s + '/' + cat + '/', f)) and is_wav(f)]
            for f in files:
                convert_to_spec_image(file_loc = files_loc, category=cat, filename=f, is_train=(s == 'train'), verbose=True)

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
'''
TODO:LQD
'''
import sys
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FC
from PyQt5.QtWidgets import QApplication, QPushButton, QMainWindow, QVBoxLayout, QWidget

class QtDraw(QMainWindow):
  flag_btn_start = True

  def __init__(self):
    super(QtDraw, self).__init__()
    self.init_ui()

  def init_ui(self):
    self.resize(800, 600)
    self.setWindowTitle('PyQt5 Draw')

    # TODO:这里是结合的关键
    self.fig = plt.Figure()
    self.canvas = FC(self.fig)
    self.btn_start = QPushButton(self)
    self.btn_start.setText('draw')
    self.btn_start.clicked.connect(self.slot_btn_start)

    widget = QWidget()
    layout = QVBoxLayout()
    layout.addWidget(self.canvas)
    layout.addWidget(self.btn_start)
    widget.setLayout(layout)
    self.setCentralWidget(widget)

  def slot_btn_start(self):
    try:
      ax = self.fig.add_subplot(111)
      x = np.linspace(0, 100, 100)
      y = np.random.random(100)
      ax.cla() # TODO:删除原图，让画布上只有新的一次的图
      ax.plot(x, y)
      self.canvas.draw() # TODO:这里开始绘制
    except Exception as e:
      print(e)

def ui_main():
  app = QApplication(sys.argv)
  w = QtDraw()
  w.show()
  sys.exit(app.exec_())

if __name__ == '__main__':
  ui_main()

max_stars_repo_name: seekindark/helloworld

max_issues_repo_name: seekindark/helloworld

max_forks_repo_name: seekindark/helloworld


text:
from __future__ import division #needed
import keras.backend as K
import numpy as np
import keras.datasets
import matplotlib.pyplot as plt
import keras
from memorization_utils import *
import os
from sklearn.linear_model import Ridge
import sys
sys.path.append('../rcvs_fexps/')
sys.path.append('../rcvs_fexps/iMIMIC-RCVs/')
sys.path.append('../rcvs_fexps/iMIMIC-RCVs/scripts/')
sys.path.append('../rcvs_fexps/iMIMIC-RCVs/scripts/keras_vis_rcv/')
#from rcv_utils import *
from mnist_utils import *
import rcv_utils
import PIL

from scipy import misc
import numpy as np
#import tensorflow as tf
import argparse
from skimage.feature import greycomatrix, greycoprops
import skimage.color

import cv2

import tensorflow as tf

"""
Here we gather all the functions used to compute the concept measures,
the measures of colorfulness, color-ness, and texuture descriptors of
the images.
"""

def compute_concept_measures(x_test):
    concept_measures={}
    concept_measures['dissimilarity']=[]
    concept_measures['contrast']=[]
    concept_measures['homogeneity']=[]
    concept_measures['ASM']=[]
    concept_measures['energy']=[]
    concept_measures['correlation']=[]
    for i in range(len(x_test)):
        glcm = greycomatrix(skimage.img_as_ubyte(skimage.color.rgb2gray(x_test[i])), [1], [0] , symmetric=True, normed=True)
        concept_measures['dissimilarity'].append(greycoprops(glcm, 'dissimilarity'))
        concept_measures['contrast'].append(greycoprops(glcm, 'contrast'))
        concept_measures['homogeneity'].append(greycoprops(glcm, 'homogeneity'))
        concept_measures['ASM'].append( greycoprops(glcm, 'ASM'))
        concept_measures['energy'].append( greycoprops(glcm, 'energy'))
        concept_measures['correlation'].append(greycoprops(glcm, 'correlation'))
    for k in concept_measures.keys():
        concept_measures[k] = np.asarray(concept_measures[k]).T[0][0]
    return concept_measures

def colorfulness(img):
    # split the image into its respective RGB components
    (B, G, R) = cv2.split(img.astype("float"))

    # compute rg = R - G
    rg = np.absolute(R - G)

    # compute yb = 0.5 * (R + G) - B
    yb = np.absolute(0.5 * (R + G) - B)

    # compute the mean and standard deviation of both `rg` and `yb`
    (rbMean, rbStd) = (np.mean(rg), np.std(rg))
    (ybMean, ybStd) = (np.mean(yb), np.std(yb))

    # combine the mean and standard deviations
    stdRoot = np.sqrt((rbStd ** 2) + (ybStd ** 2))
    meanRoot = np.sqrt((rbMean ** 2) + (ybMean ** 2))

    # derive the "colorfulness" metric and return it
    return stdRoot + (0.3 * meanRoot)

"""
modules for
      RGB to LAB color-space conversion
      LAB to RGB color-space conversion
"""

def preprocess(image):
    with tf.name_scope('preprocess'):
        # [0, 1] => [-1, 1]
        return image * 2 - 1

def deprocess(image):
    with tf.name_scope('deprocess'):
        # [-1, 1] => [0, 1]
        return (image + 1) / 2

def preprocess_lab(lab):
    with tf.name_scope('preprocess_lab'):
        L_chan, a_chan, b_chan = tf.unstack(lab, axis=2)
        # L_chan: black and white with input range [0, 100]
        # a_chan/b_chan: color channels with input range ~[-110, 110], not exact
        # [0, 100] => [-1, 1],  ~[-110, 110] => [-1, 1]
        return [L_chan / 50 - 1, a_chan / 110, b_chan / 110]

def deprocess_lab(L_chan, a_chan, b_chan):
    with tf.name_scope('deprocess_lab'):
        #TODO This is axis=3 instead of axis=2 when deprocessing batch of images
               # ( we process individual images but deprocess batches)
        #return tf.stack([(L_chan + 1) / 2 * 100, a_chan * 110, b_chan * 110], axis=3)
        return tf.stack([(L_chan + 1) / 2 * 100, a_chan * 110, b_chan * 110], axis=2)

def augment(image, brightness):
    # (a, b) color channels, combine with L channel and convert to rgb
    a_chan, b_chan = tf.unstack(image, axis=3)
    L_chan = tf.squeeze(brightness, axis=3)
    lab = deprocess_lab(L_chan, a_chan, b_chan)
    rgb = lab_to_rgb(lab)
    return rgb

def check_image(image):
    assertion = tf.assert_equal(tf.shape(image)[-1], 3, message='image must have 3 color channels')
    with tf.control_dependencies([assertion]):
        image = tf.identity(image)

    if image.get_shape().ndims not in (3, 4):
        raise ValueError('image must be either 3 or 4 dimensions')

    # make the last dimension 3 so that you can unstack the colors
    shape = list(image.get_shape())
    shape[-1] = 3
    image.set_shape(shape)
    return image

def rgb_to_lab(srgb):
    # based on https://github.com/torch/image/blob/9f65c30167b2048ecbe8b7befdc6b2d6d12baee9/generic/image.c
    with tf.name_scope('rgb_to_lab'):
        srgb = check_image(srgb)
        srgb_pixels = tf.reshape(srgb, [-1, 3])
        with tf.name_scope('srgb_to_xyz'):
            linear_mask = tf.cast(srgb_pixels <= 0.04045, dtype=tf.float32)
            exponential_mask = tf.cast(srgb_pixels > 0.04045, dtype=tf.float32)
            rgb_pixels = (srgb_pixels / 12.92 * linear_mask) + (((srgb_pixels + 0.055) / 1.055) ** 2.4) * exponential_mask
            rgb_to_xyz = tf.constant([
                #    X        Y          Z
                [0.412453, 0.212671, 0.019334], # R
                [0.357580, 0.715160, 0.119193], # G
                [0.180423, 0.072169, 0.950227], # B
            ])
            xyz_pixels = tf.matmul(rgb_pixels, rgb_to_xyz)

        # https://en.wikipedia.org/wiki/Lab_color_space#CIELAB-CIEXYZ_conversions
        with tf.name_scope('xyz_to_cielab'):
            # convert to fx = f(X/Xn), fy = f(Y/Yn), fz = f(Z/Zn)

            # normalize for D65 white point
            xyz_normalized_pixels = tf.multiply(xyz_pixels, [1/0.950456, 1.0, 1/1.088754])

            epsilon = 6/29
            linear_mask = tf.cast(xyz_normalized_pixels <= (epsilon**3), dtype=tf.float32)
            exponential_mask = tf.cast(xyz_normalized_pixels > (epsilon**3), dtype=tf.float32)
            fxfyfz_pixels = (xyz_normalized_pixels / (3 * epsilon**2) + 4/29) * linear_mask + (xyz_normalized_pixels ** (1/3)) * exponential_mask

            # convert to lab
            fxfyfz_to_lab = tf.constant([
                #  l       a       b
                [  0.0,  500.0,    0.0], # fx
                [116.0, -500.0,  200.0], # fy
                [  0.0,    0.0, -200.0], # fz
            ])
            lab_pixels = tf.matmul(fxfyfz_pixels, fxfyfz_to_lab) + tf.constant([-16.0, 0.0, 0.0])

        return tf.reshape(lab_pixels, tf.shape(srgb))

def lab_to_rgb(lab):
    with tf.name_scope('lab_to_rgb'):
        lab = check_image(lab)
        lab_pixels = tf.reshape(lab, [-1, 3])
        # https://en.wikipedia.org/wiki/Lab_color_space#CIELAB-CIEXYZ_conversions
        with tf.name_scope('cielab_to_xyz'):
            # convert to fxfyfz
            lab_to_fxfyfz = tf.constant([
                #   fx      fy        fz
                [1/116.0, 1/116.0,  1/116.0], # l
                [1/500.0,     0.0,      0.0], # a
                [    0.0,     0.0, -1/200.0], # b
            ])
            fxfyfz_pixels = tf.matmul(lab_pixels + tf.constant([16.0, 0.0, 0.0]), lab_to_fxfyfz)

            # convert to xyz
            epsilon = 6/29
            linear_mask = tf.cast(fxfyfz_pixels <= epsilon, dtype=tf.float32)
            exponential_mask = tf.cast(fxfyfz_pixels > epsilon, dtype=tf.float32)
            xyz_pixels = (3 * epsilon**2 * (fxfyfz_pixels - 4/29)) * linear_mask + (fxfyfz_pixels ** 3) * exponential_mask

            # denormalize for D65 white point
            xyz_pixels = tf.multiply(xyz_pixels, [0.950456, 1.0, 1.088754])

        with tf.name_scope('xyz_to_srgb'):
            xyz_to_rgb = tf.constant([
                #     r           g          b
                [ 3.2404542, -0.9692660,  0.0556434], # x
                [-1.5371385,  1.8760108, -0.2040259], # y
                [-0.4985314,  0.0415560,  1.0572252], # z
            ])
            rgb_pixels = tf.matmul(xyz_pixels, xyz_to_rgb)
            # avoid a slightly negative number messing up the conversion
            rgb_pixels = tf.clip_by_value(rgb_pixels, 0.0, 1.0)
            linear_mask = tf.cast(rgb_pixels <= 0.0031308, dtype=tf.float32)
            exponential_mask = tf.cast(rgb_pixels > 0.0031308, dtype=tf.float32)
            srgb_pixels = (rgb_pixels * 12.92 * linear_mask) + ((rgb_pixels ** (1/2.4) * 1.055) - 0.055) * exponential_mask

    return tf.reshape(srgb_pixels, tf.shape(lab))

def rgb_to_hsv(rgb):
    # Translated from source of colorsys.rgb_to_hsv
    # r,g,b should be a numpy arrays with values between 0 and 255
    # rgb_to_hsv returns an array of floats between 0.0 and 1.0.
    rgb = rgb.astype('float')
    hsv = np.zeros_like(rgb)
    # in case an RGBA array was passed, just copy the A channel
    hsv[..., 3:] = rgb[..., 3:]
    r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2]
    maxc = np.max(rgb[..., :3], axis=-1)
    minc = np.min(rgb[..., :3], axis=-1)
    hsv[..., 2] = maxc
    mask = maxc != minc
    hsv[mask, 1] = (maxc - minc)[mask] / maxc[mask]
    rc = np.zeros_like(r)
    gc = np.zeros_like(g)
    bc = np.zeros_like(b)
    rc[mask] = (maxc - r)[mask] / (maxc - minc)[mask]
    gc[mask] = (maxc - g)[mask] / (maxc - minc)[mask]
    bc[mask] = (maxc - b)[mask] / (maxc - minc)[mask]
    hsv[..., 0] = np.select(
        [r == maxc, g == maxc], [bc - gc, 2.0 + rc - bc], default=4.0 + gc - rc)
    hsv[..., 0] = (hsv[..., 0] / 6.0) % 1.0
    return hsv

def hsv_to_rgb(hsv):
    # Translated from source of colorsys.hsv_to_rgb
    # h,s should be a numpy arrays with values between 0.0 and 1.0
    # v should be a numpy array with values between 0.0 and 255.0
    # hsv_to_rgb returns an array of uints between 0 and 255.
    rgb = np.empty_like(hsv)
    rgb[..., 3:] = hsv[..., 3:]
    h, s, v = hsv[..., 0], hsv[..., 1], hsv[..., 2]
    i = (h * 6.0).astype('uint8')
    f = (h * 6.0) - i
    p = v * (1.0 - s)
    q = v * (1.0 - s * f)
    t = v * (1.0 - s * (1.0 - f))
    i = i % 6
    conditions = [s == 0.0, i == 1, i == 2, i == 3, i == 4, i == 5]
    rgb[..., 0] = np.select(conditions, [v, q, p, p, t, v], default=v)
    rgb[..., 1] = np.select(conditions, [v, v, v, q, p, p], default=t)
    rgb[..., 2] = np.select(conditions, [v, p, t, v, v, q], default=p)
    return rgb.astype('uint8')

from skimage.feature import greycomatrix, greycoprops
import skimage.color
# loading concept measures (only for validation split)
def compute_concept_measures(x_test):
    concept_measures={}
    concept_measures['dissimilarity']=[]
    concept_measures['contrast']=[]
    concept_measures['homogeneity']=[]
    concept_measures['ASM']=[]
    concept_measures['energy']=[]
    concept_measures['correlation']=[]

    concept_measures['colorfulness']=[]
    concept_measures['black']=[]
    concept_measures['white']=[]
    concept_measures['red']=[]
    concept_measures['orange']=[]
    concept_measures['yellow']=[]
    concept_measures['green']=[]
    concept_measures['cyano']=[]
    concept_measures['blue']=[]
    concept_measures['purple']=[]
    concept_measures['magenta']=[]

    colors_list = ['black', 'white', 'red', 'orange','yellow', 'green', 'cyano', 'blue', 'purple', 'magenta']

    for i in range(len(x_test)):
        glcm = greycomatrix(skimage.img_as_ubyte(skimage.color.rgb2gray(x_test[i])), [1], [0] , symmetric=True, normed=True)
        concept_measures['dissimilarity'].append(greycoprops(glcm, 'dissimilarity'))
        concept_measures['contrast'].append(greycoprops(glcm, 'contrast'))
        concept_measures['homogeneity'].append(greycoprops(glcm, 'homogeneity'))
        concept_measures['ASM'].append( greycoprops(glcm, 'ASM'))
        concept_measures['energy'].append( greycoprops(glcm, 'energy'))
        concept_measures['correlation'].append(greycoprops(glcm, 'correlation'))
        concept_measures['colorfulness'].append(colorfulness(x_test[i]))
        x_test[i] = cv2.cvtColor(x_test[i],cv2.COLOR_RGB2BGR)
        for color in colors_list:
            concept_measures[color].append(colorness(x_test[i], color))
    for k in ['dissimilarity', 'contrast', 'homogeneity','ASM', 'energy', 'correlation']:
        concept_measures[k] = np.asarray(concept_measures[k]).T[0][0]
    return concept_measures

def colorfulness(img):
    # split the image into its respective RGB components
    (B, G, R) = cv2.split(img.astype("float"))

    # compute rg = R - G
    rg = np.absolute(R - G)

    # compute yb = 0.5 * (R + G) - B
    yb = np.absolute(0.5 * (R + G) - B)

    # compute the mean and standard deviation of both `rg` and `yb`
    (rbMean, rbStd) = (np.mean(rg), np.std(rg))
    (ybMean, ybStd) = (np.mean(yb), np.std(yb))

    # combine the mean and standard deviations
    stdRoot = np.sqrt((rbStd ** 2) + (ybStd ** 2))
    meanRoot = np.sqrt((rbMean ** 2) + (ybMean ** 2))

    # derive the "colorfulness" metric and return it
    return stdRoot + (0.3 * meanRoot)
def hsv_histograms(image):
    hist_hue = cv2.calcHist([image], [0], None, [180], [0, 180])
    hist_sat = cv2.calcHist([image], [1], None, [256], [0, 256])
    hist_val = cv2.calcHist([image], [2], None, [256], [0, 256])
    print np.mean(image[:,:,0])
    print np.min(image[:,:,0])
    print np.max(image[:,:,0])

    return hist_hue, hist_sat, hist_val

def color_picker(color_name):
    brg_colors={}
    brg_colors['red']= np.uint8([[[0,0,255 ]]])
    brg_colors['orange'] = np.uint8([[[0,128,255 ]]])
    brg_colors['yellow'] = np.uint8([[[0,255,255 ]]])
    brg_colors['green'] = np.uint8([[[0,255,0 ]]])
    brg_colors['cyano'] = np.uint8([[[255,255,0 ]]])
    brg_colors['blue'] = np.uint8([[[255,0,0]]])
    brg_colors['purple'] = np.uint8([[[255,0,128]]])
    brg_colors['magenta'] = np.uint8([[[255,0,255 ]]])
    brg_colors['white'] = np.uint8([[[255,255,255 ]]])
    brg_colors['black'] = np.uint8([[[0,0,0 ]]])

    rgb_color_code = brg_colors[color_name]
    return cv2.cvtColor(rgb_color_code,cv2.COLOR_BGR2HSV)

def round_hue(hue_val):
    hues = np.arange(0,180)
    if hue_val<180:
        hue_def = hues[hue_val]
    else:
        hue_def = hues[(hue_val)%179]
    return hue_def

def quantize_hue_ranges(image, color_name):
    if color_name == 'red':
        hue_min = 165
        hue_max = 10

    elif color_name == 'orange':
        hue_min = 10
        hue_max = 25
    elif color_name == 'yellow':
        hue_min = 25
        hue_max = 40
    elif color_name == 'green':
        hue_min = 40
        hue_max = 75
    elif color_name == 'cyano':
        hue_min = 75
        hue_max = 100
    elif color_name == 'blue':
        hue_min = 100
        hue_max = 125
    elif color_name == 'purple':
        hue_min = 125
        hue_max = 145
    elif color_name == 'magenta':
        hue_min = 145
        hue_max = 165
    elif (color_name == 'white' or color_name == 'black'):
        hue_min = 0
        hue_max = 255

    return hue_min, hue_max

def colorness(image, color_name, threshold = 0, verbose=False):
    image = cv2.cvtColor(image,cv2.COLOR_BGR2HSV)
    #plt.imshow(image)
    if verbose:
        h,s,v = hsv_histograms(image)
        plt.figure()
        plt.plot(h)
        plt.figure()
        plt.plot(s)
        plt.figure()
        plt.plot(v)

    if threshold == 0:
        hue_min, hue_max = quantize_hue_ranges(image, color_name)
        if verbose:
            print 'hue min, hue max: ', hue_min, hue_max
    else:
        h_point =color_picker(color_name)
        hue_min = round_hue(h_point[0][0][0]-threshold)
        hue_max = round_hue(h_point[0][0][0]+threshold)
        if verbose:
            print 'hue min, hue max: ', hue_min, hue_max
    '''
    if verbose:
        print 'red', color_picker(brg_colors['red'])
        print 'orange', color_picker(brg_colors['orange'])
        print 'yellow', color_picker(brg_colors['yellow'])
        print 'green', color_picker(brg_colors['green'])
        print 'cyano', color_picker(brg_colors['cyano'])
        print 'blue', color_picker(brg_colors['blue'])
        print 'purple', color_picker(brg_colors['purple'])
        print  'magenta', color_picker(brg_colors['magenta'])
    '''
    if (hue_min == hue_max == 0) or (hue_min == 0 and hue_max == 255):
        #it is either black or white
        if color_name=='black':
            low_c = np.array([0,
                              0,
                              0])
            upp_c = np.array([hue_max,
                              100,
                              100])
        if color_name=='white':
            low_c = np.array([0,
                              0,
                              190])
            upp_c = np.array([hue_max,
                              50,
                              255])

        if verbose:
            print 'low_c', low_c, 'upp_c', upp_c

        mask = cv2.inRange(image, low_c, upp_c)

    elif hue_min>hue_max:
        low_c = np.array([0,
                      50,
                      77])
        upp_c = np.array([hue_max,
                      255,
                      255])
        mask1 = cv2.inRange(image, low_c, upp_c)

        low_c = np.array([hue_min,
                      50,
                      77])
        upp_c = np.array([180,
                      255,
                      255])
        mask2 = cv2.inRange(image, low_c, upp_c)

        mask = cv2.bitwise_or(mask1, mask1, mask2)

    else:
        low_c = np.array([hue_min,
                          50,
                          77])
        upp_c = np.array([hue_max,
                          255,
                          255])
        if verbose:
            print 'low_c', low_c, 'upp_c', upp_c

        mask = cv2.inRange(image, low_c, upp_c)
    if verbose:
        print mask

    res = cv2.bitwise_and(image, image, mask = mask)
    if verbose:
        plt.figure()
        plt.imshow(mask, cmap='Greys')
        plt.colorbar()

        plt.figure()
        plt.imshow(cv2.cvtColor(image, cv2.COLOR_HSV2RGB))
        plt.figure()
        plt.imshow(cv2.cvtColor(res, cv2.COLOR_HSV2RGB))

    #print np.sum(mask==255)
    x,y,z = image.shape
    print x, y , z
    if verbose:
        print np.sum(mask==255)/(float(x)*float(y))

    return float(np.sum(mask==255))/(float(x)*float(y))

def get_color_mask(image, color_name, threshold=0, verbose=False):
    image = cv2.cvtColor(image,cv2.COLOR_RGB2HSV)

    if threshold == 0:
        hue_min, hue_max = quantize_hue_ranges(image, color_name)
        if verbose:
            print 'hue min, hue max: ', hue_min, hue_max
    else:
        h_point =color_picker(color_name)
        hue_min = round_hue(h_point[0][0][0]-threshold)
        hue_max = round_hue(h_point[0][0][0]+threshold)
        if verbose:
            print 'hue min, hue max: ', hue_min, hue_max
    '''
    if verbose:
        print 'red', color_picker(brg_colors['red'])
        print 'orange', color_picker(brg_colors['orange'])
        print 'yellow', color_picker(brg_colors['yellow'])
        print 'green', color_picker(brg_colors['green'])
        print 'cyano', color_picker(brg_colors['cyano'])
        print 'blue', color_picker(brg_colors['blue'])
        print 'purple', color_picker(brg_colors['purple'])
        print  'magenta', color_picker(brg_colors['magenta'])
    '''
    if (hue_min == hue_max == 0) or (hue_min == 0 and hue_max == 255):
        print 'black or white', color_name
        #it is either black or white
        if color_name=='black':
            print 'in black'
            low_c = np.array([0,
                              0,
                              0])
            upp_c = np.array([hue_max,
                              100,
                              100])
        if color_name=='white':
            print 'in white'
            low_c = np.array([0,
                              0,
                              190])
            upp_c = np.array([hue_max,
                              50,
                              255])

        mask = cv2.inRange(image, low_c, upp_c)
        return mask
    if hue_min>hue_max:
        low_c = np.array([0,
                      50,
                      77])
        upp_c = np.array([hue_max,
                      255,
                      255])
        mask1 = cv2.inRange(image, low_c, upp_c)

        low_c = np.array([hue_min,
                      50,
                      77])
        upp_c = np.array([180,
                      255,
                      255])
        mask2 = cv2.inRange(image, low_c, upp_c)

        mask = cv2.bitwise_or(mask1, mask1, mask2)

    else:
        low_c = np.array([hue_min,
                          50,
                          77])
        upp_c = np.array([hue_max,
                          255,
                          255])
        if verbose:
            print 'low_c', low_c, 'upp_c', upp_c

        mask = cv2.inRange(image, low_c, upp_c)
    if verbose:
        print mask
    return mask

def image_colors(image, threshold=0):
    image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
    x,y,z = image.shape
    new_image = np.ones((x,y,3), dtype='uint8')*255
    #print new_image
    #new_image = cv2.cvtColor(new_image,cv2.COLOR_BGR2RGB)
    #plt.imshow(new_image)
    #new_image = cv2.cvtColor(image,cv2.COLOR_RGB2HSV)
    colors = ['black', 'white', 'red', 'orange','yellow', 'green', 'cyano', 'blue', 'purple', 'magenta']
    #plt.imshow(image)

    for c in colors:
        #print c
        color_mask = get_color_mask(image, c, threshold=threshold)
        #plt.figure()
        #plt.imshow(color_mask)
        #print color_mask
        #plt.figure()
        #plt.title(c)
        #new_image =  cv2.cvtColor(new_image,cv2.COLOR_RGB2HSV)
        new_image = recolor(new_image, color_mask, c)
        #new_image =  cv2.cvtColor(new_image,cv2.COLOR_HSV2RGB)
        #plt.imshow(new_image)
        #plt.figure()
        #plt.imshow(color_mask)

        #
    new_image =  cv2.cvtColor(new_image,cv2.COLOR_HSV2RGB)
    plt.figure()
    plt.imshow(new_image)
    #print new_image
    return new_image

def recolor(image, color_mask, color_name):
    color_hues = {}
    color_hues['red']=5
    color_hues['orange']=15
    color_hues['yellow']=30
    color_hues['green']=50
    color_hues['cyano']=90
    color_hues['blue']=120
    color_hues['purple']=140
    color_hues['magenta']=160
    color_hues['black']=0
    color_hues['white']=0

    if color_name == 'black':
        image[color_mask==255] = (0,0,0)

    elif color_name == 'white':
        image[color_mask==255] = (0,0,255)
    else:
        #print image[color_mask==255].shape
        image[color_mask==255] = (color_hues[color_name],255,255)
        #print image.shape

        #plt.imshow(image)
    return image


text:
# coding=utf-8
# Copyright 2021 The Robustness Metrics Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""A zero-shot CLIP classification model.

For more information on the models please refer to

  https://openai.com/blog/clip/

and the PyTorch documentation: https://github.com/openai/CLIP/

You need `CLIP` to be on sys.path before loading this module. Note that the
model will be downloaded on the first run.

We are using a single prompt for each query: "This is a photo of a {label}".
The labels can be found in `imagenet.json`, in the same directory as this file.
"""
import json
import os

import clip
import numpy as np
from robustness_metrics.common import pipeline_builder
import torch

def create(network):
  """Loads the CLIP model."""
  json_path = os.path.join(os.path.dirname(__file__), "imagenet.json")
  with open(json_path, "r") as fp:
    imagenet_labels = json.load(fp)

  with torch.set_grad_enabled(False):
    model, _ = clip.load(network, device="cuda", jit=False)
    model = model.eval()

  prompts = clip.tokenize([
      f"This is a photo of a {label}" for label in imagenet_labels])

  with torch.no_grad():
    prompts_features = model.encode_text(prompts.cuda()).float()
    prompts_features /= prompts_features.norm(dim=-1, keepdim=True)

  image_mean = [0.48145466, 0.4578275, 0.40821073]
  image_std = [0.26862954, 0.26130258, 0.27577711]

  def call(features):
    # Normalize according to the documentation. Note that the pre-processing
    # will already have the range normalized to [0, 1].
    images_normalized = (features["image"] - image_mean) / image_std
    # Reshape from [batch, h, w, c] -> [batch, c, h, w]
    images_torch = torch.tensor(
        np.transpose(images_normalized, [0, 3, 1, 2]).astype(np.float32))
    with torch.no_grad():
      image_features = model.encode_image(images_torch.to("cuda")).float()
      image_features /= image_features.norm(dim=-1, keepdim=True)
      similarities = image_features @ prompts_features.T
      # The 100 (inv temperature) comes from the released code.
      return (100.0 * similarities).softmax(dim=-1).cpu().numpy()

  input_resolution = model.visual.input_resolution
  preprocess_config = (f"resize_small({input_resolution})|"
                       f"central_crop({input_resolution})|"
                       f"value_range(0,1)")
  preprocess_fn = pipeline_builder.get_preprocess_fn(
      preprocess_config, remove_tpu_dtypes=False)
  return call, preprocess_fn