text:
import numpy as np
import random as rand
import math, csv

def read_from_file():
	with open("data-2.txt", 'r') as f:
		data = csv.reader(f)
		data =  list(data)
	return data

def read_from_file1():
	with open("data-1.txt", 'r') as f:
		data = csv.reader(f)
		data =  list(data)
	return data

def temp_print_to_file(data):
	with open("cluster.csv", "w") as f:
		f. write("cluster1\n")
		for line in data[0]:
			line = ','.join(line)
			f.write(line)
			f.write('\n')
		f.write("cluster2\n")
		for line in data[1]:
			line = ','.join(line)
			f.write(line)
			f.write('\n')

def euclidean_distance(point1,point2):
	total = 0;
	for i in range(0, len(point1)):
		total = total + pow((float(point1[i])-float(point2[i])),2);
	total = math.sqrt(total)
	return total

def find_SSE(points,center):
	error = 0.
	for x in range(0,len(points)):
		for y in range(0,len(points[x])):
			for z in range(0, len(points[x][y])):
				error = error + pow(float(points[x][y][z]) - float(center[x][z]),2);
	return error;

def kmeans(data, k, threshold):
	center = [];
	with open("result"+str(k)+".txt", "w") as f:
		do = 0;
		f.write("k = "+ str(k)+"\n");
		for i in range(0,k):
				center.append(data[rand.randint(0, len(data))]);
		while(do < threshold):
			newcenter = [];
			points = [];
			change = 0
			for i in range(0,k):
				points.append([]);
				newcenter.append([]);
			temp = data[0]
			for x in range(0,len(data)):
				addto = k+1;
				mindistance = 999999999999999999999;
				for y in range(0,k):
					if(mindistance > euclidean_distance(center[y],data[x])):
						mindistance = euclidean_distance(center[y],data[x])
						addto = y;
				#print(addto);
				points[addto].append(data[x]);
			#print("Error : " + str(find_SSE(points,center)))
			f.write(str(find_SSE(points,center)));
			f.write("\n")
			#for x in range (0,k):
				#print(len(points[x]))

			#temp_print_to_file(points)
			for x in range(0, k):
				TotalInArea = [0]*len(data[0])
				if(len(points[x])==0):
					for a in range(0,k):
						print(len(points[a]), k)
				for y in range(0,len(points[x])):
					for i in range(0,784):
						TotalInArea[i] = TotalInArea[i] + float(points[x][y][i])
				for i in range(0,784):
						newcenter[x].append(TotalInArea[i]/len(points[x]));

			for x in range(0, k):
				similarity = 0
				for y in range(0,784):
					#print("newcenter = "+str(newcenter[x][y]))
					#print("oldcenter = "+str(center[x][y]))
					if(newcenter[x][y]-float(center[x][y])==0):
						similarity= similarity + 1
					#	print ("sim")
				#print (similarity)
				if(similarity == 784):
					change= change+1
				#print("change ="+str(change))
			if (change == k):
				return center
			else:
				center = newcenter
				do = do+1

class Node:

	def __init__(self, points, self_id, pair_id):
		self.points = points
		self.self_id = self_id
		self.height = 0

def single_link_dist(clusters, calc):
	min_dist = 999999999999999999999
	lhs, rhs = None, None
	for i in range(len(clusters)):
		for j in range(i + 1, len(clusters)):
			# print '---', len(clusters[i].self_id), len(clusters[j].self_id)
			# min_dist = 999999999999999999999
			# for a in clusters[i].self_id:
			# 	for b in clusters[j].self_id:
			# 		if min_dist > calc[a][b]:
			# 			min_dist = calc[a][b]
			# 			lhs, rhs = i, j
			tmp_dist = calc[clusters[i].self_id[0]][clusters[j].self_id[0]]
			if tmp_dist < min_dist:
				min_dist = tmp_dist
				lhs, rhs = i, j
	return min_dist, lhs, rhs

def complete_link_dist(clusters, calc):
	max_dist = 0
	lhs, rhs = None, None
	for i in range(len(clusters)):
		for j in range(i + 1, len(clusters)):
			# tmp_dist = 0
			# for a in clusters[i].self_id:
			# 	for b in clusters[j].self_id:
			# 		if tmp_dist < calc[a][b]:
			# 			tmp_dist = calc[a][b]
			tmp_dist = calc[clusters[i].self_id[0]][clusters[j].self_id[0]]
			if tmp_dist > max_dist:
				max_dist = tmp_dist
				lhs, rhs = i, j
	return max_dist, lhs, rhs

def merge_cluster(cluster1,cluster2, min_dist):
	# print 'merge:', cluster2.self_id
	for i in range(len(cluster2.self_id)):
		cluster1.points.append(cluster2.points[i])
		cluster1.self_id.append(cluster2.self_id[i])
	cluster1.height = min_dist

def init_cluster(clusters):
	nodes = []
	for i in range(len(clusters)):
		nodes.append(Node([clusters[i]], [i], 0))
	calc = [];
	for i in range(len(clusters)):
		calc.append([]);
		for j in range(len(clusters)):
			calc[i].append(euclidean_distance(clusters[i],clusters[j]));
	return nodes, calc

def hac_single_link(clusters):
	nodes, calc = init_cluster(clusters)
	while len(nodes) > 1:
		min_dist, lhs, rhs = single_link_dist(nodes, calc)
		merge_cluster(nodes[lhs], nodes[rhs], min_dist)
		del nodes[rhs]
		if(len(nodes) == 10):
			print ("============== Single Link HAC ==============")
		if(len(nodes)<= 10):
			for i in range(len(nodes)):
				print(nodes[i].self_id, nodes[i].height)
			print("==========next============");

def hac_complete_link(clusters):
	nodes, calc = init_cluster(clusters)
	while len(nodes) > 1:
		max_dist, lhs, rhs = complete_link_dist(nodes, calc)
		merge_cluster(nodes[lhs], nodes[rhs], max_dist)
		del nodes[rhs]

		if(len(nodes) == 10):
			print ("============== Complete Link HAC ==============")
		if(len(nodes)<= 10):
			for i in range(len(nodes)):
				print(nodes[i].self_id, nodes[i].height)
			print("==========next============");

c = [
	    [123,    312,     434,     4325,   345345],
	    [23124,  141241,  434234,  9837489, 34743],
	    [128937, 127,     12381,   424,      8945],
	    [323,    4348,    5040,    8189,     2348],
	    [51249,  42190,   2713,    2319,     4328],
	    [13957,  1871829, 8712847, 34589,   30945],
	    [1234,   45094,   23409,   13495,  348052],
	    [49853,  3847,    4728,    4059,     5389]
	]

if __name__ == "__main__":
	# data =  read_from_file()
	# for i in range(2, 11):
	# 	kmeans(data, i, 1000000)
	data =  read_from_file()
	hac_single_link(data)
	hac_complete_link(data)

ext: py

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""reV tech mapping framework.

This module manages the exclusions-to-resource mapping.
The core of this module is a parallel cKDTree.

Created on Fri Jun 21 16:05:47 2019

@author: gbuster
"""
from concurrent.futures import as_completed
import h5py
import logging
from math import ceil
import numpy as np
import os
from scipy.spatial import cKDTree
from warnings import warn

from reV.supply_curve.extent import SupplyCurveExtent
from reV.utilities.exceptions import FileInputWarning, FileInputError

from rex.resource import Resource
from rex.utilities.execution import SpawnProcessPool
from rex.utilities.utilities import res_dist_threshold

logger = logging.getLogger(__name__)

class TechMapping:
    """Framework to create map between tech layer (exclusions), res, and gen"""

    def __init__(self, excl_fpath, res_fpath, sc_resolution=2560,
                 dist_margin=1.05):
        """
        Parameters
        ----------
        excl_fpath : str
            Filepath to exclusions h5 file, must contain latitude and longitude
            arrays to allow for mapping to resource points
        res_fpath : str
            Filepath to .h5 resource file that we're mapping to.
        sc_resolution : int | None, optional
            Supply curve resolution, does not affect the exclusion to resource
            (tech) mapping, but defines how many exclusion pixels are mapped
            at a time, by default 2560
        dist_margin : float, optional
            Extra margin to multiply times the computed distance between
            neighboring resource points, by default 1.05
        """
        self._excl_fpath = excl_fpath
        self._check_fout()

        self._tree, self._dist_thresh = \
            self._build_tree(res_fpath, dist_margin=dist_margin)

        with SupplyCurveExtent(self._excl_fpath,
                               resolution=sc_resolution) as sc:
            self._sc_resolution = sc.resolution
            self._gids = np.array(list(range(len(sc))), dtype=np.uint32)
            self._excl_shape = sc.exclusions.shape
            self._n_excl = np.product(self._excl_shape)
            self._sc_row_indices = sc.row_indices
            self._sc_col_indices = sc.col_indices
            self._excl_row_slices = sc.excl_row_slices
            self._excl_col_slices = sc.excl_col_slices
            logger.info('Initialized TechMapping object with {} calc chunks '
                        'for {} tech exclusion points'
                        .format(len(self._gids), self._n_excl))

    @property
    def distance_threshold(self):
        """Get the upper bound on NN distance between excl and res points.

        Returns
        -------
        float
            Estimate the distance between resource points. Calculated as half
            of the diagonal between closest resource points, with desired
            extra margin
        """
        return self._dist_thresh

    @staticmethod
    def _build_tree(res_fpath, dist_margin=1.05):
        """
        Build cKDTree from resource lat, lon coordinates. Compute minimum
        intra point distance between resource gids with provided extra margin.

        Parameters
        ----------
        res_fpath : str
            Filepath to .h5 resource file that we're mapping to.
        dist_margin : float, optional
            Extra margin to multiply times the computed distance between
            neighboring resource points, by default 1.05

        Returns
        -------
        tree : cKDTree
            cKDTree built from resource lat, lon coordinates
        dist_tresh : float
            Estimate the distance between resource points. Calculated as half
            of the diagonal between closest resource points, with desired
            extra margin
        """
        with Resource(res_fpath) as f:
            lat_lons = f.lat_lon

        # pylint: disable=not-callable
        tree = cKDTree(lat_lons)

        dist_thresh = res_dist_threshold(lat_lons, tree=tree,
                                         margin=dist_margin)

        return tree, dist_thresh

    @staticmethod
    def _make_excl_iarr(shape):
        """
        Create 2D array of 1D index values for the flattened h5 excl extent

        Parameters
        ----------
        shape : tuple
            exclusion extent shape

        Returns
        -------
        iarr : ndarray
            2D array of 1D index values for the flattened h5 excl extent
        """
        iarr = np.arange(np.product(shape), dtype=np.uint32)

        return iarr.reshape(shape)

    @staticmethod
    def _get_excl_slices(gid, sc_row_indices, sc_col_indices, excl_row_slices,
                         excl_col_slices):
        """
        Get the row and column slices of the exclusions grid corresponding
        to the supply curve point gid.

        Parameters
        ----------
        gid : int
            Supply curve point gid.
        sc_row_indices : list
            List of row indices in exclusion array for for every sc_point gid
        sc_col_indices : list
            List of column indices in exclusion array for for every sc_point
            gid
        excl_row_slices : list
            List representing the supply curve points rows. Each list entry
            contains the exclusion row slice that are included in the sc
            point.
        excl_col_slices : list
            List representing the supply curve points columns. Each list entry
            contains the exclusion columns slice that are included in the sc
            point.

        Returns
        -------
        row_slice : int
            Exclusions grid row index slice corresponding to the sc point gid.
        col_slice : int
            Exclusions grid col index slice corresponding to the sc point gid.
        """

        row_slice = excl_row_slices[sc_row_indices[gid]]
        col_slice = excl_col_slices[sc_col_indices[gid]]

        return row_slice, col_slice

    @classmethod
    def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices,
                         excl_row_slices, excl_col_slices,
                         coord_labels=('latitude', 'longitude')):
        """
        Extract the exclusion coordinates for teh desired gids for TechMapping.

        Parameters
        ----------
        gids : np.ndarray
            Supply curve gids with tech exclusion points to map to the
            resource meta points.
        excl_fpath : str
            Filepath to exclusions h5 file, must contain latitude and longitude
            arrays to allow for mapping to resource points
        sc_row_indices : list
            List of row indices in exclusion array for for every sc_point gid
        sc_col_indices : list
            List of column indices in exclusion array for for every sc_point
            gid
        excl_row_slices : list
            List representing the supply curve points rows. Each list entry
            contains the exclusion row slice that are included in the sc
            point.
        excl_col_slices : list
            List representing the supply curve points columns. Each list entry
            contains the exclusion columns slice that are included in the sc
            point.
        coord_labels : tuple
            Labels for the coordinate datasets.

        Returns
        -------
        coords_out : list
            List of arrays of the un-projected latitude, longitude array of
            tech exclusion points. List entries correspond to input gids.
        """
        coords_out = []
        with h5py.File(excl_fpath, 'r') as f:
            for gid in gids:
                row_slice, col_slice = cls._get_excl_slices(gid,
                                                            sc_row_indices,
                                                            sc_col_indices,
                                                            excl_row_slices,
                                                            excl_col_slices)
                try:
                    lats = f[coord_labels[0]][row_slice, col_slice]
                    lons = f[coord_labels[1]][row_slice, col_slice]
                    emeta = np.vstack((lats.flatten(), lons.flatten())).T
                except Exception as e:
                    m = ('Could not unpack coordinates for gid {} with '
                         'row/col slice {}/{}. Received the following '
                         'error:\n{}'.format(gid, row_slice, col_slice, e))
                    logger.error(m)
                    raise e

                coords_out.append(emeta)

        return coords_out

    @classmethod
    def map_resource_gids(cls, gids, excl_fpath, sc_row_indices,
                          sc_col_indices, excl_row_slices, excl_col_slices,
                          tree, dist_thresh):
        """Map exclusion gids to the resource meta.

        Parameters
        ----------
        gids : np.ndarray
            Supply curve gids with tech exclusion points to map to the
            resource meta points.
        excl_fpath : str
            Filepath to exclusions h5 file, must contain latitude and longitude
            arrays to allow for mapping to resource points
        sc_row_indices : list
            List of row indices in exclusion array for for every sc_point gid
        sc_col_indices : list
            List of column indices in exclusion array for for every sc_point
            gid
        excl_row_slices : list
            List representing the supply curve points rows. Each list entry
            contains the exclusion row slice that are included in the sc
            point.
        excl_col_slices : list
            List representing the supply curve points columns. Each list entry
            contains the exclusion columns slice that are included in the sc
            point.
        tree : cKDTree
            cKDTree built from resource lat, lon coordinates
        dist_tresh : float
            Estimate the distance between resource points. Calculated as half
            of the diagonal between closest resource points, with an extra
            5% margin

        Returns
        -------
        ind : list
            List of arrays of index values from the NN. List entries correspond
            to input gids.
        """
        logger.debug('Getting tech map coordinates for chunks {} through {}'
                     .format(gids[0], gids[-1]))
        ind_out = []
        coords_out = cls._get_excl_coords(excl_fpath, gids, sc_row_indices,
                                          sc_col_indices, excl_row_slices,
                                          excl_col_slices)

        logger.debug('Running tech mapping for chunks {} through {}'
                     .format(gids[0], gids[-1]))
        for i, _ in enumerate(gids):
            dist, ind = tree.query(coords_out[i])
            ind[(dist >= dist_thresh)] = -1
            ind_out.append(ind)

        return ind_out

    @staticmethod
    def save_tech_map(excl_fpath, dset, indices, distance_threshold=None,
                      res_fpath=None, chunks=(128, 128)):
        """Save tech mapping indices and coordinates to an h5 output file.

        Parameters
        ----------
        excl_fpath : str
            Filepath to exclusions h5 file to add techmap to as 'dset'
        dset : str
            Dataset name in fpath_out to save mapping results to.
        indices : np.ndarray
            Index values of the NN resource point. -1 if no res point found.
            2D integer array with shape equal to the exclusions extent shape.
        distance_threshold : float
            Distance upper bound to save as attr.
        res_fpath : str, optional
            Filepath to .h5 resource file that we're mapping to,
            by default None
        chunks : tuple
            Chunk shape of the 2D output datasets.
        """
        logger.info('Writing tech map "{}" to {}'.format(dset, excl_fpath))

        shape = indices.shape
        chunks = (np.min((shape[0], chunks[0])), np.min((shape[1], chunks[1])))

        with h5py.File(excl_fpath, 'a') as f:
            if dset in list(f):
                wmsg = ('TechMap results dataset "{}" is being replaced '
                        'in pre-existing Exclusions TechMapping file "{}"'
                        .format(dset, excl_fpath))
                logger.warning(wmsg)
                warn(wmsg, FileInputWarning)
                f[dset][...] = indices
            else:
                f.create_dataset(dset, shape=shape, dtype=indices.dtype,
                                 data=indices, chunks=chunks)

            if distance_threshold:
                f[dset].attrs['distance_threshold'] = distance_threshold

            if res_fpath:
                f[dset].attrs['src_res_fpath'] = res_fpath

        logger.info('Successfully saved tech map "{}" to {}'
                    .format(dset, excl_fpath))

    def _check_fout(self):
        """Check the TechMapping output file for cached data."""
        with h5py.File(self._excl_fpath, 'r') as f:
            if 'latitude' not in f or 'longitude' not in f:
                emsg = ('Datasets "latitude" and/or "longitude" not in '
                        'pre-existing Exclusions TechMapping file "{}". '
                        'Cannot proceed.'
                        .format(os.path.basename(self._excl_fpath)))
                logger.exception(emsg)
                raise FileInputError(emsg)

    def map_resource(self, max_workers=None, points_per_worker=10):
        """
        Map all resource gids to exclusion gids

        Parameters
        ----------
        max_workers : int, optional
            Number of cores to run mapping on. None uses all available cpus,
            by default None
        points_per_worker : int, optional
            Number of supply curve points to map to resource gids on each
            worker, by default 10

        Returns
        -------
        indices : np.ndarray
            Index values of the NN resource point. -1 if no res point found.
            2D integer array with shape equal to the exclusions extent shape.
        """
        gid_chunks = ceil(len(self._gids) / points_per_worker)
        gid_chunks = np.array_split(self._gids, gid_chunks)

        # init full output arrays
        indices = -1 * np.ones((self._n_excl, ), dtype=np.int32)
        iarr = self._make_excl_iarr(self._excl_shape)

        futures = {}
        loggers = [__name__, 'reV']
        with SpawnProcessPool(max_workers=max_workers,
                              loggers=loggers) as exe:

            # iterate through split executions, submitting each to worker
            for i, gid_set in enumerate(gid_chunks):
                # submit executions and append to futures list
                futures[exe.submit(self.map_resource_gids,
                                   gid_set,
                                   self._excl_fpath,
                                   self._sc_row_indices,
                                   self._sc_col_indices,
                                   self._excl_row_slices,
                                   self._excl_col_slices,
                                   self._tree,
                                   self.distance_threshold)] = i

            n_finished = 0
            for future in as_completed(futures):
                n_finished += 1
                logger.info('Parallel TechMapping futures collected: '
                            '{} out of {}'
                            .format(n_finished, len(futures)))

                i = futures[future]
                result = future.result()

                for j, gid in enumerate(gid_chunks[i]):
                    row_slice, col_slice = self._get_excl_slices(
                        gid,
                        self._sc_row_indices,
                        self._sc_col_indices,
                        self._excl_row_slices,
                        self._excl_col_slices)
                    ind_slice = iarr[row_slice, col_slice].flatten()
                    indices[ind_slice] = result[j]

        indices = indices.reshape(self._excl_shape)

        return indices

    @classmethod
    def run(cls, excl_fpath, res_fpath, dset=None, sc_resolution=2560,
            dist_margin=1.05, max_workers=None, points_per_worker=10):
        """Run parallel mapping and save to h5 file.

        Parameters
        ----------
        excl_fpath : str
            Filepath to exclusions h5 (tech layer). dset will be
            created in excl_fpath.
        res_fpath : str
            Filepath to .h5 resource file that we're mapping to.
        dset : str, optional
            Dataset name in excl_fpath to save mapping results to, if None
            do not save tech_map to excl_fpath, by default None
        sc_resolution : int | None, optional
            Supply curve resolution, does not affect the exclusion to resource
            (tech) mapping, but defines how many exclusion pixels are mapped
            at a time, by default 2560
        dist_margin : float, optional
            Extra margin to multiply times the computed distance between
            neighboring resource points, by default 1.05
        max_workers : int, optional
            Number of cores to run mapping on. None uses all available cpus,
            by default None
        points_per_worker : int, optional
            Number of supply curve points to map to resource gids on each
            worker, by default 10

        Returns
        -------
        indices : np.ndarray
            Index values of the NN resource point. -1 if no res point found.
            2D integer array with shape equal to the exclusions extent shape.
        """
        kwargs = {"dist_margin": dist_margin,
                  "sc_resolution": sc_resolution}
        mapper = cls(excl_fpath, res_fpath, **kwargs)
        indices = mapper.map_resource(max_workers=max_workers,
                                      points_per_worker=points_per_worker)

        if dset:
            mapper.save_tech_map(excl_fpath, dset, indices,
                                 distance_threshold=mapper.distance_threshold,
                                 res_fpath=res_fpath)

        return indices

max_stars_repo_name: pjstanle/reV

max_issues_repo_name: pjstanle/reV

max_forks_repo_name: pjstanle/reV

reason: import numpy,from scipy


text:
#!/usr/bin/env python

# Resource: https://github.com/npinto/opencv/tree/master/samples/python2
# Laura Petrich and Chris Hegberg
import rospy
from std_msgs.msg import String, Bool
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError
import numpy as np
import cv2

class TagDetector:
    def __init__(self):
        FLANN_INDEX_KDTREE = 1
        FLANN_INDEX_LSH    = 6
        flann_params= dict(algorithm = FLANN_INDEX_LSH,
                            table_number = 6, # 12
                            key_size = 12,     # 20
                            multi_probe_level = 1) #2
        UA_IMG = cv2.imread('/home/hegberg/catkin_ws/src/competition_4/src/competition_4/UA_100.png')
        AR_IMG = cv2.imread('/home/hegberg/catkin_ws/src/competition_4/src/competition_4/ar.png')
        self.ua_init = False
        self.ar_init = False
        self.detector = cv2.ORB( nfeatures = 1000 )
        self.match_finder = cv2.FlannBasedMatcher(flann_params, {})

        while not self.ua_init:
            self.ua_kp1, descrs = self.detector.detectAndCompute(UA_IMG, None)
            if descrs != None:
                self.ua_init = True
                self.ua_d1 = descrs
                print 'UA descriptors found'
        while not self.ar_init:
            self.ar_kp1, descrs = self.detector.detectAndCompute(AR_IMG, None)
            if descrs != None:
                self.ar_init = True
                self.ar_d1 = descrs
                print 'AR descriptors found'

        self.bridge = CvBridge()
        self.mode = 0 # default searching mode
        self.ua_rect = (0, 0, 360, 480)
        self.ar_rect = (0, 0, 180, 178)
        self.logo = None
        self.frame = None
        self.ar_verts = np.float32([[0, 0, 0], [0, 1, 0], [1, 1, 0], [1, 0, 0],
                               [0, 0, 1], [0, 1, 1], [1, 1, 1], [1, 0, 1],
                               [0, 0.5, 2], [1, 0.5, 2]])
        self.house_edges = [(0, 1), (1, 2), (2, 3), (3, 0),
                    (4, 5), (5, 6), (6, 7), (7, 4),
                    (0, 4), (1, 5), (2, 6), (3, 7),
                    (4, 8), (5, 8), (6, 9), (7, 9), (8, 9)]
        # calibrated values for our turtlebot
        self.K = np.float64([[ 545.91605983,    0.        ,  306.35546651],
               [   0.        ,  542.20157959,  244.21889428],
               [   0.        ,    0.        ,    1.        ]])

        self.D = np.float64([ 0.03880064, -0.11735783,  0.00824685, -0.00252129,  0.        ])
        self.MIN_MATCH_COUNT = 10

    def detect_loop(self):
        current_frame = self.frame
        pos_msg = ''
        tag = 'NONE'
        if current_frame != None:
            kp2, d2 = self.detector.detectAndCompute(current_frame, None)
            if len(kp2) < 30 or d2 == None:
                return current_frame, pos_msg, tag

            ua_current_frame, ua_pos_msg, ua_flag, ua_length = self.match_work(kp2, d2, current_frame, pos_msg, 0)
            ar_current_frame, ar_pos_msg, ar_flag, ar_length = self.match_work(kp2, d2, current_frame, pos_msg, 1)

            if ua_flag and ua_length > ar_length:
                # found UA tag
                current_frame = ua_current_frame
                pos_msg = ua_pos_msg
                tag = 'UA'
            elif ar_flag and ar_length > 25:
                # found AR tag
                current_frame = ar_current_frame
                pos_msg = ar_pos_msg
                tag = 'AR'
        return current_frame, pos_msg, tag

    def match_work(self, kp2, d2, current_frame, pos_msg, mode):
        if mode == 0: # UA tag
            matches = self.match_finder.knnMatch(d2, self.ua_d1, k = 2)
            x0, y0, x1, y1 = self.ua_rect
            kp1 = self.ua_kp1
        else: # AR tag
            matches = self.match_finder.knnMatch(d2, self.ar_d1, k = 2)
            x0, y0, x1, y1 = (70, 70, 110, 110)
            kp1 = self.ar_kp1

        matches = [m[0] for m in matches if len(m) == 2 and m[0].distance < m[1].distance * 0.75]
        l = len(matches)
        if l < self.MIN_MATCH_COUNT:
            return current_frame, pos_msg, False, 0
        matches_by_id = [[] for _ in xrange(1)]
        for m in matches:
            matches_by_id[m.imgIdx].append(m)
        for _, matches in enumerate(matches_by_id):
            if len(matches) < self.MIN_MATCH_COUNT:
                continue
            p0 = [kp1[m.trainIdx].pt for m in matches]
            p1 = [kp2[m.queryIdx].pt for m in matches]
            p0, p1 = np.float32((p0, p1))
            H, status = cv2.findHomography(p0, p1, cv2.RANSAC, 3.0)
            status = status.ravel() != 0
            if status.sum() < self.MIN_MATCH_COUNT:
                continue
            p0, p1 = p0[status], p1[status]
            for (x, y) in np.int32(p1):
                cv2.circle(current_frame, (x, y), 2, (0, 0, 255))
            quad = np.float32([[x0, y0], [x1, y0], [x1, y1], [x0, y1]])
            quad = cv2.perspectiveTransform(quad.reshape(1, -1, 2), H).reshape(-1, 2)
            quad3D = np.float32([[x0, y0, 0], [x1, y0, 0], [x1, y1, 0], [x0, y1, 0]])
            ret, rvec, tvec = cv2.solvePnP(quad3D, quad, self.K, self.D)
            verts = self.ar_verts * [(x1-x0), (y1-y0), -(x1-x0)*0.3] + (x0, y0, 0)
            imgPts = cv2.projectPoints(verts, rvec, tvec, self.K, self.D)[0].reshape(-1, 2)
            for i in range(4):
                (x, y) = imgPts[i]
                # cv2.circle(current_frame, (int(x), int(y)), 10, (255, 0, 0), -1)
                pos_msg += str(int(x)) + ' ' + str(int(y)) + ' '
        return current_frame, pos_msg, True, l

    def img_callback(self, data):
        if self.mode == 0:
            try:
                self.frame = self.bridge.imgmsg_to_cv2(data, "bgr8")
            except CvBridgeError as e:
                print e

    def img_callback2(self, data):
        if self.mode == 1:
            # docking, switch to front camera
            try:
                self.frame = self.bridge.imgmsg_to_cv2(data, "bgr8")
            except CvBridgeError as e:
                print e

    def dock_callback(self, data):
        check = data.data
        if check:
            self.mode = 1
        else:
            self.mode = 0

    def dock_callback2(self, data):
        check = data.data
        if check:
            self.mode = 0
        else:
            self.mode = 1

    def already_found_callback(self, data):
        check = data.data
        if check:
            self.mode = 0

if __name__ == '__main__':
    rospy.init_node('tag_detector')
    t = TagDetector()
    cv2.namedWindow('TAG_DETECTOR')
    img_sub = rospy.Subscriber('/cam1/camera/image_raw', Image, t.img_callback, queue_size = 1, buff_size=2**24)
    img_sub2 = rospy.Subscriber('/camera/rgb/image_raw', Image, t.img_callback2, queue_size = 1, buff_size=2**24)
    # img_sub2 = rospy.Subscriber('/cam2/image_raw', Image, t.img_callback2, queue_size = 10)
    mode_sub = rospy.Subscriber('docking_start', Bool, t.dock_callback, queue_size = 10)
    mode_sub2 = rospy.Subscriber('docking_done', Bool, t.dock_callback2, queue_size = 10)
    already_found_sub = rospy.Subscriber('already_found', Bool, t.already_found_callback, queue_size = 5)
    pub_l = rospy.Publisher('target_location', String, queue_size=5)
    pub_c = rospy.Publisher('tag_control', String, queue_size=5)
    r = rospy.Rate(10)
    while True:
        tag = 'NONE'
        current_frame, pos_msg, tag = t.detect_loop()
        if tag != 'NONE' and len(pos_msg) > 0:
            pub_l.publish(pos_msg)
            pub_c.publish(tag)
        if current_frame != None:
            cv2.imshow('TAG_DETECTOR', current_frame)
            ch = cv2.waitKey(1)
            if ch == 27:
                break
        r.sleep()

max_stars_repo_name: Hegberg/DaffyDackDivision

max_issues_repo_name: Hegberg/DaffyDackDivision

max_forks_repo_name: Hegberg/DaffyDackDivision


text:
# coding=utf-8
# Copyright 2019 The SEED Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r"""SEED actor."""

import os

from absl import flags
from absl import logging
import numpy as np
from seed_rl import grpc
from seed_rl.common import common_flags
from seed_rl.common import profiling
from seed_rl.common import utils
import tensorflow as tf

FLAGS = flags.FLAGS

flags.DEFINE_integer('task', 0, 'Task id.')
flags.DEFINE_integer('num_actors_with_summaries', 4,
                     'Number of actors that will log debug/profiling TF '
                     'summaries.')
flags.DEFINE_bool('render', False,
                  'Whether the first actor should render the environment.')

def are_summaries_enabled():
  return FLAGS.task < FLAGS.num_actors_with_summaries

def is_rendering_enabled():
  return FLAGS.render and FLAGS.task == 0

def actor_loop(create_env_fn):
  """Main actor loop.

  Args:
    create_env_fn: Callable (taking the task ID as argument) that must return a
      newly created environment.
  """
  logging.info('Starting actor loop')
  if are_summaries_enabled():
    summary_writer = tf.summary.create_file_writer(
        os.path.join(FLAGS.logdir, 'actor_{}'.format(FLAGS.task)),
        flush_millis=20000, max_queue=1000)
    timer_cls = profiling.ExportingTimer
  else:
    summary_writer = tf.summary.create_noop_writer()
    timer_cls = utils.nullcontext

  actor_step = 0
  with summary_writer.as_default():
    while True:
      try:
        # Client to communicate with the learner.
        client = grpc.Client(FLAGS.server_address)

        env = create_env_fn(FLAGS.task)

        # Unique ID to identify a specific run of an actor.
        run_id = np.random.randint(np.iinfo(np.int64).max)
        observation = env.reset()
        reward = 0.0
        raw_reward = 0.0
        done = False

        episode_step = 0
        episode_return = 0
        episode_raw_return = 0

        while True:
          tf.summary.experimental.set_step(actor_step)
          env_output = utils.EnvOutput(reward, done, observation)
          with timer_cls('actor/elapsed_inference_s', 1000):
            action, curr_state, curr_action = client.inference(
                FLAGS.task, run_id, env_output, raw_reward)

          # if FLAGS.task == 3:
          # if actor_step < 100 and FLAGS.task == 3 and episode_step < 1:
          #   logging.info('State: {} Action: {} Observation: {} Reward: {} Done: {}'.format(curr_state[0][0][0], curr_action, observation[:,:,0], reward, done))
          # if actor_step < 100 and FLAGS.task == 3 and episode_step == 1:
          #   logging.info('State: {} Action: {}'.format(curr_state[0][0][0], curr_action))

          with timer_cls('actor/elapsed_env_step_s', 1000):
            observation, reward, done, info = env.step(action.numpy())
          if is_rendering_enabled():
            env.render()
          episode_step += 1
          episode_return += reward
          raw_reward = float((info or {}).get('score_reward', reward))
          episode_raw_return += raw_reward
          if done:
            logging.info('Return: %f Raw return: %f Steps: %i', episode_return,
                         episode_raw_return, episode_step)
            with timer_cls('actor/elapsed_env_reset_s', 10):
              observation = env.reset()
              episode_step = 0
              episode_return = 0
              episode_raw_return = 0
            if is_rendering_enabled():
              env.render()
          actor_step += 1
      except (tf.errors.UnavailableError, tf.errors.CancelledError) as e:
        logging.exception(e)
        env.close()


text:
"""Class to analyse a GROMACS trajectory"""

import numpy as np
import matplotlib.pyplot as plt

import MDAnalysis as mda
import MDAnalysis.analysis.align
import pandas as pd

class GmxAnalysis:
    def __init__(self):
        # Class variables to be defined by child class:
        self.u = None
        self.selection_parser = {
            'protein': """protein""",
            'backbone': """name CA or name C or name N""",
            'alpha_C': """name CA""",
            'heavy': "protein and not name H*",
            'alkane_ua': """name C*"""
        }

    def load_trajectory(self, structfile, trajfile=None):
        """
        Loads trajectory from the given structure and trajectory files.

        Args:
            structfile (str): Path to GROMACS structure file (.gro or .tpr).
            structfile (str): Path to GROMACS trajectory file (.trr or .xtc) [if None, then only loads structure].
        """
        if trajfile is None:
            return mda.Universe(structfile)
        else:
            return mda.Universe(structfile, trajfile)

    def _rgyr(self, coords, masses):
        r"""
        Calculates the radius of gyration of atoms with given coordinates and
        masses.

        If $r_i(t)$ is the selection of coordinates,
        $$R_g = \sqrt{\frac{1}{N} \sum_{k=1}^{N} (r_k - r_{com})^2 }$$

        Args:
            coords (np.array): Array of shape (N, 3) containing atomic coordinates.
            masses (np.array): Array of shape (N,) containing atomic masses.

        Returns:
            Radius of gyration (np.float).

        Raises:
            ValueError if coords does not have the right shape, or if the
            lengths of coords and masses do not match.
        """
        com = np.average(coords, weights=masses, axis=0)
        sq_distances = np.sum((coords - com)**2, axis=1)
        Rg = np.sqrt(np.average(sq_distances, weights=masses))
        return Rg

    def radius_of_gyration(self, u=None, selection=None):
        """
        Calculates radius of gyration of selection along trajectory.

        Args:
            u (mda.Universe): Universe (default=self.u, i.e. main universe).
            selection (str): MDAnalysis selection string (default=None, i.e. all atoms).

        Returns:
            Time-indexed radii of gyration (pd.DataFrame)
        """
        if u is None:
            u = self.u

        times = []
        Rgs = []
        sel = u.select_atoms(selection)

        for ts in u.trajectory:
            Rgval = self._rgyr(sel.positions, sel.masses)
            times.append(ts.time)
            Rgs.append(Rgval)

        times = np.array(times)
        Rgs = np.array(Rgs)

        return pd.DataFrame({"t": times, "Rg": Rgs})

    def plot_data(self, df):
        """Plots data from dataframe

        Args:
            df: Dataframe to plot. Index will be plotted on the

        Returns:
            tuple(fig, ax):
                fig: Matplotlib figure object
                ax: Matplotlib axis object corresponding to the figure
        """
        fig, ax = plt.subplots(dpi=150)
        cols = df.columns.astype(str)
        ax.plot(df[cols[0]], df[cols[1]])
        ax.set_xlabel(cols[0])
        ax.set_ylabel(cols[1])

        return fig, ax


text:
import torch
#
torch.multiprocessing.set_sharing_strategy('file_system')

import pandas as pd
import numpy as np
import sys
import os
import pickle
import argparse

# local imports
import train_model
import experiment_routines as experiments
from subsetting_exp import subset_experiment

# can add different
from dataset import get_data_loaders

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('group_key', type=str,
                        help='which key to subset on')
    parser.add_argument('--num_seeds', type=int, default=5,
                        help='number of seeds to run')
    parser.add_argument('--seed_beginning', type=int, default=0,
                        help='seed to start with')
    parser.add_argument('--dataset_name', type=str, default='isic',
                        help='seed to start with')

    args = parser.parse_args()
    group_key = args.group_key
    num_seeds, seed_beginning = args.num_seeds, args.seed_beginning
    dataset_name = args.dataset_name

    fracs_group_a = [0.02, 0.05, 0.2, 0.5, 0.8, 0.95,  0.98]

    learning_rates = [0.01, 0.001, 0.0001]
    weight_decays = [.01, .001, 0.0001]
#    momentums = [0, 0.9]
    momentums = [0.9]
#    number_of_epochs = [10,20]
    number_of_epochs = [20]

    if dataset_name == 'isic':
        label_colname_ = 'benign_malignant_01'
        original_data_csv_ = 'isic/fresh/int_data/df_no_sonic.csv'
        image_fp_ = 'isic/fresh/Data/Images'
        all_group_colnames_ = ['study_name_id',
                                'age_approx_by_decade_id',
                                'age_over_45_id',
                                'age_over_50_id',
                                'age_over_60_id',
                                'anatom_site_general_id',
                                'sex_id']
        results_descriptor_ ='isic_hpSel'

    elif dataset_name == 'cifar':

        if (group_key == "air"):
            label_colname_ = "animal"
        else:
            label_colname_ = "air"
        original_data_csv_ = 'cifar4/df_cifar4_labels.csv'
        image_fp_ = 'cifar4/images'
        all_group_colnames_ = ["air", "animal"]
        results_descriptor_ ='cifar4_subsetting_hpSel_debug'

    else:
        print('TODO: need to input the data and image files')

    for lr in learning_rates:
            for wd in weight_decays:
                for momentum in momentums:
                        for num_epochs in number_of_epochs:
                            sgd_params = {'lr': lr, 'weight_decay': wd, 'momentum': momentum}

                            subset_experiment(num_seeds,
                                              seed_beginning,
                                              group_key,
                                              label_colname = label_colname_,
                                              original_data_csv = original_data_csv_,
                                              image_fp = image_fp_,
                                              all_group_colnames = all_group_colnames_,
                                              fracs_group_a = fracs_group_a,
                                              eval_key='val',
                                              results_descriptor = results_descriptor_,
                                              num_epochs=num_epochs,
                                              sgd_params = sgd_params,
                                              )

max_stars_repo_name: estherrolf/representation-matters

max_issues_repo_name: estherrolf/representation-matters

max_forks_repo_name: estherrolf/representation-matters


text:
#!/usr/bin/env python
"""Convolutional variational auto-encoder for binarized MNIST.

The neural networks are written with TensorFlow Slim.

References
----------
http://edwardlib.org/tutorials/decoder
http://edwardlib.org/tutorials/inference-networks
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import edward as ed
import numpy as np
import os
import tensorflow as tf

from edward.models import Bernoulli, Normal
from edward.util import Progbar
from observations import mnist
from scipy.misc import imsave
from tensorflow.contrib import slim

def generator(array, batch_size):
  """Generate batch with respect to array's first axis."""
  start = 0  # pointer to where we are in iteration
  while True:
    stop = start + batch_size
    diff = stop - array.shape[0]
    if diff <= 0:
      batch = array[start:stop]
      start += batch_size
    else:
      batch = np.concatenate((array[start:], array[:diff]))
      start = diff
    batch = batch.astype(np.float32) / 255.0  # normalize pixel intensities
    batch = np.random.binomial(1, batch)  # binarize images
    yield batch

def generative_network(z):
  """Generative network to parameterize generative model. It takes
  latent variables as input and outputs the likelihood parameters.

  logits = neural_network(z)
  """
  with slim.arg_scope([slim.conv2d_transpose],
                      activation_fn=tf.nn.elu,
                      normalizer_fn=slim.batch_norm,
                      normalizer_params={'scale': True}):
    net = tf.reshape(z, [M, 1, 1, d])
    net = slim.conv2d_transpose(net, 128, 3, padding='VALID')
    net = slim.conv2d_transpose(net, 64, 5, padding='VALID')
    net = slim.conv2d_transpose(net, 32, 5, stride=2)
    net = slim.conv2d_transpose(net, 1, 5, stride=2, activation_fn=None)
    net = slim.flatten(net)
    return net

def inference_network(x):
  """Inference network to parameterize variational model. It takes
  data as input and outputs the variational parameters.

  loc, scale = neural_network(x)
  """
  with slim.arg_scope([slim.conv2d, slim.fully_connected],
                      activation_fn=tf.nn.elu,
                      normalizer_fn=slim.batch_norm,
                      normalizer_params={'scale': True}):
    net = tf.reshape(x, [M, 28, 28, 1])
    net = slim.conv2d(net, 32, 5, stride=2)
    net = slim.conv2d(net, 64, 5, stride=2)
    net = slim.conv2d(net, 128, 5, padding='VALID')
    net = slim.dropout(net, 0.9)
    net = slim.flatten(net)
    params = slim.fully_connected(net, d * 2, activation_fn=None)

  loc = params[:, :d]
  scale = tf.nn.softplus(params[:, d:])
  return loc, scale

ed.set_seed(42)

data_dir = "/tmp/data"
out_dir = "/tmp/out"
if not os.path.exists(out_dir):
  os.makedirs(out_dir)
M = 128  # batch size during training
d = 10  # latent dimension

# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(data_dir)
x_train_generator = generator(x_train, M)

# MODEL
z = Normal(loc=tf.zeros([M, d]), scale=tf.ones([M, d]))
logits = generative_network(z)
x = Bernoulli(logits=logits)

# INFERENCE
x_ph = tf.placeholder(tf.int32, [M, 28 * 28])
loc, scale = inference_network(tf.cast(x_ph, tf.float32))
qz = Normal(loc=loc, scale=scale)

# Bind p(x, z) and q(z | x) to the same placeholder for x.
data = {x: x_ph}
inference = ed.KLqp({z: qz}, data)
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)

hidden_rep = tf.sigmoid(logits)

tf.global_variables_initializer().run()

n_epoch = 100
n_iter_per_epoch = x_train.shape[0] // M
for epoch in range(1, n_epoch + 1):
  print("Epoch: {0}".format(epoch))
  avg_loss = 0.0

  pbar = Progbar(n_iter_per_epoch)
  for t in range(1, n_iter_per_epoch + 1):
    pbar.update(t)
    x_batch = next(x_train_generator)
    info_dict = inference.update(feed_dict={x_ph: x_batch})
    avg_loss += info_dict['loss']

  # Print a lower bound to the average marginal likelihood for an
  # image.
  avg_loss = avg_loss / n_iter_per_epoch
  avg_loss = avg_loss / M
  print("-log p(x) <= {:0.3f}".format(avg_loss))

  # Visualize hidden representations.
  images = hidden_rep.eval()
  for m in range(M):
    imsave(os.path.join(out_dir, '%d.png') % m, images[m].reshape(28, 28))

max_stars_repo_name: xiangze/edward

max_issues_repo_name: xiangze/edward

max_forks_repo_name: xiangze/edward


text:
# Copyright (c) 2020 - for information on the respective copyright owner
# see the NOTICE file and/or the repository https://github.com/boschresearch/blackboxopt
#
# SPDX-License-Identifier: Apache-2.0

import numpy as np
import pytest

from blackboxopt import Objective
from blackboxopt.utils import get_loss_vector

@pytest.mark.parametrize(
    "known,reported,expected",
    [
        (
            [Objective("score", True), Objective("loss", False)],
            {"loss": 2.0, "score": 1.0},
            [-1.0, 2.0],
        ),
        (
            [Objective("score", True), Objective("loss", False)],
            {"loss": 2.0, "score": None},
            [np.nan, 2.0],
        ),
    ],
)
def test_get_loss_vector(known, reported, expected):
    loss_vector = get_loss_vector(known_objectives=known, reported_objectives=reported)
    np.testing.assert_array_equal(loss_vector, np.array(expected))

max_stars_repo_name: boschresearch/blackboxopt

max_issues_repo_name: boschresearch/blackboxopt

max_forks_repo_name: boschresearch/blackboxopt


text:
# =============================================================================
# Copyright 2020 NVIDIA. All Rights Reserved.
# Copyright 2019 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =============================================================================

"""
This file contains code artifacts adapted from the original implementation:
https://github.com/google-research/google-research/blob/master/schema_guided_dst/baseline/data_utils.py
"""

import collections
import json
import os
import pickle
import re

import numpy as np
import torch

from nemo.collections.nlp.data.datasets.sgd_dataset.input_example import InputExample
from nemo.utils import logging

__all__ = ['FILE_RANGES', 'PER_FRAME_OUTPUT_FILENAME', 'SGDDataProcessor']

FILE_RANGES = {
    "sgd_single_domain": {"train": range(1, 44), "dev": range(1, 8), "test": range(1, 12)},
    "sgd_multi_domain": {"train": range(44, 128), "dev": range(8, 21), "test": range(12, 35)},
    "sgd_all": {"train": range(1, 128), "dev": range(1, 21), "test": range(1, 35)},
    "multiwoz": {"train": range(1, 18), "dev": range(1, 3), "test": range(1, 3)},
    "debug_sample": {"train": range(1, 2), "dev": range(1, 2), "test": range(1, 2)},
}

# Name of the file containing all predictions and their corresponding frame metrics.
PER_FRAME_OUTPUT_FILENAME = "dialogues_and_metrics.json"

class SGDDataProcessor(object):
    """Data generator for SGD dialogues."""

    def __init__(
        self, task_name, data_dir, dialogues_example_dir, tokenizer, schema_emb_processor, overwrite_dial_files=False,
    ):
        """
        Constructs SGD8DataProcessor
        Args:
            task_name (str): task  name, for  example, "single_domain"
            data_dir (str): path to data directory
            dialogues_example_dir (str): path to  store processed dialogue examples
            tokenizer (Tokenizer): such as NemoBertTokenizer
            schema_emb_processor (Obj): contains information about schemas
            overwrite_dial_files (bool): whether to overwite dialogue files
        """
        self.data_dir = data_dir
        self.dialogues_examples_dir = dialogues_example_dir

        self._task_name = task_name
        self.schema_config = schema_emb_processor.schema_config
        self.schema_emb_processor = schema_emb_processor

        train_file_range = FILE_RANGES[task_name]["train"]
        dev_file_range = FILE_RANGES[task_name]["dev"]
        test_file_range = FILE_RANGES[task_name]["test"]

        self._file_ranges = {
            "train": train_file_range,
            "dev": dev_file_range,
            "test": test_file_range,
        }

        self._seen_services = {
            "train": set(),
            "dev": set(),
            "test": set(),
        }

        self._tokenizer = tokenizer
        self._max_seq_length = self.schema_config["MAX_SEQ_LENGTH"]

        self.dial_files = {}

        # slots_relation_list.np would contain the candidate list of slots for each (service, slot) which would be
        # looked into when a switch between two services happens in the dialogue and we can not find any value for a slot in the current user utterance.
        # This file would get generated from the dialogues in the training set.
        self.slots_relation_file = os.path.join(dialogues_example_dir, f"{task_name}_train_slots_relation_list.np")

        master_device = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
        for dataset in ["train", "dev", "test"]:
            # Process dialogue files
            dial_file = f"{task_name}_{dataset}_examples.processed"
            dial_file = os.path.join(dialogues_example_dir, dial_file)
            self.dial_files[(task_name, dataset)] = dial_file

            dialog_paths = SGDDataProcessor.get_dialogue_files(data_dir, dataset, task_name)
            dialogs = SGDDataProcessor.load_dialogues(dialog_paths)
            for dialog in dialogs:
                self._seen_services[dataset].update(set(dialog['services']))

            if not os.path.exists(dial_file) or overwrite_dial_files:
                logging.debug(f"Start generating the dialogue examples for {dataset} dataset.")
                if master_device:
                    if not os.path.exists(dialogues_example_dir):
                        os.makedirs(dialogues_example_dir)
                    dial_examples, slots_relation_list = self._generate_dialog_examples(
                        dataset, schema_emb_processor.schemas
                    )
                    with open(dial_file, "wb") as f:
                        np.save(f, dial_examples)

                    if dataset == "train":
                        with open(self.slots_relation_file, "wb") as f:
                            pickle.dump(slots_relation_list, f)
                        logging.debug(
                            f"The slot carry-over list for train set is stored at {self.slots_relation_file}"
                        )

                    logging.debug(f"The dialogue examples for {dataset} dataset saved at {dial_file}")
                logging.debug(f"Finish generating the dialogue examples for {dataset} dataset.")

            # wait until the master process writes to the dialogue processed file
            if torch.distributed.is_initialized():
                torch.distributed.barrier()

    def get_dialog_examples(self, dataset):
        """
        Returns a list of `InputExample`s of the data splits' dialogues.
        Args:
          dataset(str): can be "train", "dev", or "test".
        Returns:
          examples: a list of `InputExample`s.
        """
        if (self._task_name, dataset) not in self.dial_files or not os.path.exists(
            self.dial_files[(self._task_name, dataset)]
        ):
            raise ValueError(
                f"{dataset} dialogue examples were not processed for {self._task_name} task. Re-initialize SGDDataProcessor and add {dataset} dataset to datasets arg."
            )
        dial_file = self.dial_files[(self._task_name, dataset)]
        logging.info(f"Loading dialogue examples from {dial_file}.")

        with open(dial_file, "rb") as f:
            dial_examples = np.load(f, allow_pickle=True)
            f.close()

        if not os.path.exists(self.slots_relation_file):
            raise ValueError(
                f"Slots relation file {self.slots_relation_file} does not exist. It is needed for the carry-over mechanism of state tracker for switches between services."
            )

        with open(self.slots_relation_file, "rb") as f:
            self.schema_emb_processor.update_slots_relation_list(pickle.load(f))
        logging.info(
            f"Loaded the slot relation list for value carry-over between services from {self.slots_relation_file}."
        )

        return dial_examples

    def get_seen_services(self, dataset_split):
        return self._seen_services[dataset_split]

    def _generate_dialog_examples(self, dataset, schemas):
        """
        Returns a list of `InputExample`s of the data splits' dialogues.
        Args:
          dataset(str): can be "train", "dev", or "test".
          schemas(Schema): for all services and all datasets processed by the schema_processor
        Returns:
          examples: a list of `InputExample`s.
        """
        logging.info(f'Creating examples and slot relation list from the dialogues started...')
        dialog_paths = [
            os.path.join(self.data_dir, dataset, "dialogues_{:03d}.json".format(i)) for i in self._file_ranges[dataset]
        ]
        dialogs = SGDDataProcessor.load_dialogues(dialog_paths)

        examples = []
        slot_carryover_candlist = collections.defaultdict(int)
        for dialog_idx, dialog in enumerate(dialogs):
            if dialog_idx % 1000 == 0:
                logging.info(f'Processed {dialog_idx} dialogues.')
            examples.extend(self._create_examples_from_dialog(dialog, schemas, dataset, slot_carryover_candlist))

        slots_relation_list = collections.defaultdict(list)
        for slots_relation, relation_size in slot_carryover_candlist.items():
            if relation_size > 0:
                slots_relation_list[(slots_relation[0], slots_relation[1])].append(
                    (slots_relation[2], slots_relation[3], relation_size)
                )
                slots_relation_list[(slots_relation[2], slots_relation[3])].append(
                    (slots_relation[0], slots_relation[1], relation_size)
                )

        return examples, slots_relation_list

    def _create_examples_from_dialog(self, dialog, schemas, dataset, slot_carryover_candlist):
        """
        Create examples for every turn in the dialog.
        Args:
            dialog (dict): dialogue example
            schemas(Schema): for all services and all datasets processed by the schema_processor
            dataset(str): can be "train", "dev", or "test".
            slot_carryover_candlist(dict): a dictionary to keep and count the number of carry-over cases between two slots from two different services
        Returns:
            examples: a list of `InputExample`s.
        """
        dialog_id = dialog["dialogue_id"]
        prev_states = {}
        examples = []
        for turn_idx, turn in enumerate(dialog["turns"]):
            # Generate an example for every frame in every user turn.
            if turn["speaker"] == "USER":
                user_utterance = turn["utterance"]
                user_frames = {f["service"]: f for f in turn["frames"]}
                if turn_idx > 0:
                    system_turn = dialog["turns"][turn_idx - 1]
                    system_utterance = system_turn["utterance"]
                    system_frames = {f["service"]: f for f in system_turn["frames"]}
                else:
                    system_utterance = ""
                    system_frames = {}

                turn_id = "{}-{}-{:02d}".format(dataset, dialog_id, turn_idx)
                turn_examples, prev_states, slot_carryover_values = self._create_examples_from_turn(
                    turn_id, system_utterance, user_utterance, system_frames, user_frames, prev_states, schemas
                )
                examples.extend(turn_examples)

                for value, slots_list in slot_carryover_values.items():
                    if value in ["True", "False"]:
                        continue
                    if len(slots_list) > 1:
                        for service1, slot1 in slots_list:
                            for service2, slot2 in slots_list:
                                if service1 == service2:
                                    continue
                                if service1 > service2:
                                    service1, service2 = service2, service1
                                    slot1, slot2 = slot2, slot1
                                slot_carryover_candlist[(service1, slot1, service2, slot2)] += 1
        return examples

    def _get_state_update(self, current_state, prev_state):
        """
        Updates dialogue state
        Args:
            current_state (dict): dict of slot - slot values pairs for the current dialogue turn
            prev_state (dict): dict of slot - slot values pairs for the previous dialogue turns
        Returns:
            state_update (dict): dict of slot - slot values pairs that very added/updated during the current
                dialogue turn
        """
        state_update = dict(current_state)
        for slot, values in current_state.items():
            if slot in prev_state and prev_state[slot][0] in values:
                # Remove the slot from state if its value didn't change.
                state_update.pop(slot)
        return state_update

    def _create_examples_from_turn(
        self, turn_id, system_utterance, user_utterance, system_frames, user_frames, prev_states, schemas
    ):
        """
        Creates an example for each frame in the user turn.
        Args:
            turn_id (int): turn number
            system_utterance (str): last system utterance
            user_utterance (str): lst user utterance
            system_frames (dict): all system utterances and slot - slot value pairs
            user_frames (dict): all user utterances and slot - slot value pairs
            prev_states (dict): slot - slot value pairs from the previous turns
            schemas (obj): carries information about the service from the current turn
        Returns:
            examples: a list of `InputExample`s.
            prev_states (dict): updated dialogue state
        """
        system_tokens, system_alignments, system_inv_alignments = self._tokenize(system_utterance)
        user_tokens, user_alignments, user_inv_alignments = self._tokenize(user_utterance)
        states = {}
        base_example = InputExample(schema_config=self.schema_config, is_real_example=True, tokenizer=self._tokenizer,)
        base_example.example_id = turn_id

        _, dialog_id, turn_id_ = turn_id.split('-')
        dialog_id_1, dialog_id_2 = dialog_id.split('_')
        base_example.example_id_num = [int(dialog_id_1), int(dialog_id_2), int(turn_id_)]
        base_example.add_utterance_features(
            system_tokens, system_inv_alignments, user_tokens, user_inv_alignments, system_utterance, user_utterance
        )

        examples = []
        slot_carryover_values = collections.defaultdict(list)
        for service, user_frame in user_frames.items():
            # Create an example for this service.
            example = base_example.make_copy_with_utterance_features()

            example.example_id = "{}-{}".format(turn_id, service)
            _, dialog_id, turn_id_ = turn_id.split('-')
            dialog_id_1, dialog_id_2 = dialog_id.split('_')
            example.example_id_num = [
                int(dialog_id_1),
                int(dialog_id_2),
                int(turn_id_),
                schemas.get_service_id(service),
            ]

            example.service_schema = schemas.get_service_schema(service)
            system_frame = system_frames.get(service, None)
            state = user_frame["state"]["slot_values"]
            state_update = self._get_state_update(state, prev_states.get(service, {}))
            states[service] = state

            # Populate features in the example.
            example.add_categorical_slots(state_update)
            # The input tokens to bert are in the format [CLS] [S1] [S2] ... [SEP]
            # [U1] [U2] ... [SEP] [PAD] ... [PAD]. For system token indices a bias of
            # 1 is added for the [CLS] token and for user tokens a bias of 2 +
            # len(system_tokens) is added to account for [CLS], system tokens and
            # [SEP].
            user_span_boundaries = self._find_subword_indices(
                state_update, user_utterance, user_frame["slots"], user_alignments, user_tokens, 2 + len(system_tokens)
            )
            if system_frame is not None:
                system_span_boundaries = self._find_subword_indices(
                    state_update, system_utterance, system_frame["slots"], system_alignments, system_tokens, 1
                )
            else:
                system_span_boundaries = {}
            example.add_noncategorical_slots(state_update, user_span_boundaries, system_span_boundaries)
            example.add_requested_slots(user_frame)
            example.add_intents(user_frame)
            examples.append(example)

            if service not in prev_states and int(turn_id_) > 0:
                for slot_name, values in state_update.items():
                    for value in values:
                        slot_carryover_values[value].append((service, slot_name))
                for prev_service, prev_slot_value_list in prev_states.items():
                    if prev_service == service:
                        continue
                    if prev_service in state:
                        prev_slot_value_list = state[prev_service]
                    for prev_slot_name, prev_values in prev_slot_value_list.items():
                        for prev_value in prev_values:
                            slot_carryover_values[prev_value].append((prev_service, prev_slot_name))

        return examples, states, slot_carryover_values

    def _find_subword_indices(self, slot_values, utterance, char_slot_spans, alignments, subwords, bias):
        """Find indices for subwords corresponding to slot values."""
        span_boundaries = {}
        for slot, values in slot_values.items():
            # Get all values present in the utterance for the specified slot.
            value_char_spans = {}
            for slot_span in char_slot_spans:
                if slot_span["slot"] == slot:
                    value = utterance[slot_span["start"] : slot_span["exclusive_end"]]
                    start_tok_idx = alignments[slot_span["start"]]
                    end_tok_idx = alignments[slot_span["exclusive_end"] - 1]
                    if 0 <= start_tok_idx < len(subwords):
                        end_tok_idx = min(end_tok_idx, len(subwords) - 1)
                        value_char_spans[value] = (start_tok_idx + bias, end_tok_idx + bias)
            for v in values:
                if v in value_char_spans:
                    span_boundaries[slot] = value_char_spans[v]
                    break
        return span_boundaries

    def _tokenize(self, utterance):
        """Tokenize the utterance using word-piece tokenization used by BERT.

        Args:
          utterance: A string containing the utterance to be tokenized.

        Returns:
          bert_tokens: A list of tokens obtained by word-piece tokenization of the
            utterance.
          alignments: A dict mapping indices of characters corresponding to start
            and end positions of words (not subwords) to corresponding indices in
            bert_tokens list.
          inverse_alignments: A list of size equal to bert_tokens. Each element is a
            tuple containing the index of the starting and inclusive ending
            character of the word corresponding to the subword. This list is used
            during inference to map word-piece indices to spans in the original
            utterance.
        """
        # utterance = tokenization.convert_to_unicode(utterance)

        # After _naive_tokenize, spaces and punctuation marks are all retained, i.e.
        # direct concatenation of all the tokens in the sequence will be the
        # original string.
        tokens = SGDDataProcessor._naive_tokenize(utterance)
        # Filter out empty tokens and obtain aligned character index for each token.
        alignments = {}
        char_index = 0
        bert_tokens = []
        # These lists store inverse alignments to be used during inference.
        bert_tokens_start_chars = []
        bert_tokens_end_chars = []
        for token in tokens:
            if token.strip():
                subwords = self._tokenizer.text_to_tokens(token)
                # Store the alignment for the index of starting character and the
                # inclusive ending character of the token.
                alignments[char_index] = len(bert_tokens)
                bert_tokens_start_chars.extend([char_index] * len(subwords))
                bert_tokens.extend(subwords)
                # The inclusive ending character index corresponding to the word.
                inclusive_char_end = char_index + len(token) - 1
                alignments[inclusive_char_end] = len(bert_tokens) - 1
                bert_tokens_end_chars.extend([inclusive_char_end] * len(subwords))
            char_index += len(token)
        inverse_alignments = list(zip(bert_tokens_start_chars, bert_tokens_end_chars))
        return bert_tokens, alignments, inverse_alignments

    def get_num_dialog_examples(self, dataset):
        """
        Gets the number of dilaog examples in the data split.
        Args:
          dataset: str. can be "train", "dev", or "test".
        Returns:from nemo_nlp.data.datasets.sgd import data_utils
          example_count: int. number of examples in the specified dataset.
        """
        example_count = 0
        dialog_paths = [
            os.path.join(self.data_dir, dataset, "dialogues_{:03d}.json".format(i)) for i in self._file_ranges[dataset]
        ]
        dst_set = SGDDataProcessor.load_dialogues(dialog_paths)
        for dialog in dst_set:
            for turn in dialog["turns"]:
                if turn["speaker"] == "USER":
                    example_count += len(turn["frames"])
        return example_count

    @classmethod
    def _naive_tokenize(cls, s):
        """
        Tokenizes a string, separating words, spaces and punctuations.
        Args:
            s (str): a string
        Returns:
            seq_tok (list): list of words, spaces and punctuations from the s
        """
        # Spaces and punctuation marks are all retained, i.e. direct concatenation
        # of all the tokens in the sequence will be the original string.
        seq_tok = [tok for tok in re.split(r"([^a-zA-Z0-9])", s) if tok]
        return seq_tok

    @classmethod
    def load_dialogues(cls, dialog_json_filepaths):
        """
        Obtain the list of all dialogues from specified json files.
        Args:
            dialog_json_filepaths (list): list of json files
        Returns:
            dialogs  (list): the list of all dialogues
        """
        dialogs = []
        for dialog_json_filepath in sorted(dialog_json_filepaths):
            with open(dialog_json_filepath, 'r') as f:
                dialogs.extend(json.load(f))
                f.close()
        return dialogs

    @classmethod
    def get_dialogue_files(cls, data_dir, dataset_split, task_name):
        """
        Obtain the list of all dialogue json files
        Args:
            data_dir (str): path to the data folde
            dataset_split (str): dev, test or train
            task_name (str): SGD task name, see keys of the FILE_RANGES
        Returns:
            dialogs (list): the list of all dialogue json files paths
        """
        return [
            os.path.join(data_dir, dataset_split, 'dialogues_{:03d}.json'.format(fid))
            for fid in FILE_RANGES[task_name][dataset_split]
        ]

max_stars_repo_name: ekmb/NeMo

max_issues_repo_name: okuchaiev/NeMo

max_forks_repo_name: okuchaiev/NeMo


text:
from numpy import exp

##################Initial Conditions and Physical Parameters##################
t0=0; dt=0.005;k01=0.1; k02=1; R=8.314; E1=7000; E2=20000;
V=1000; cP=-100; cR=5; cT=5; cO=1400;
##############################################################################

def Ctfxn(Cr0,Ci0,Cp0,T,tf):
    k1=k01*exp(-E1/(R*T)); k2=k02*exp(-E2/(R*T)); k1r=k1/10; n=int((tf-t0)/dt);
    Cr=Cr0*1; Ci=Ci0*1; Cp=Cp0*1; t=t0*1;

    def dCrdt(t,Cr,Ci,k1,k1r):
        return -k1*Cr+k1r*Ci
    def dCidt(t,Cr,Ci,k1,k1r,k2):
        return k1*Cr-k1r*Ci-k2*Ci
    def dCpdt(t,Ci,k2):
        return k2*Ci

    for i in range(n):
        k1R=dt*dCrdt(t,Cr,Ci,k1,k1r)
        k1I=dt*dCidt(t,Cr,Ci,k1,k1r,k2)
        k1P=dt*dCpdt(t,Ci,k2)
        k2R=dt*dCrdt(t+dt,Cr+k1R/2,Ci+k1I/2,k1,k1r)
        k2I=dt*dCidt(t+dt,Cr+k1R/2,Ci+k1I/2,k1,k1r,k2)
        k2P=dt*dCpdt(t+dt,Ci+k1I/2,k2)
        k3R=dt*dCrdt(t+dt,Cr+k2R/2,Ci+k2I/2,k1,k1r)
        k3I=dt*dCidt(t+dt,Cr+k2R/2,Ci+k2I/2,k1,k1r,k2)
        k3P=dt*dCpdt(t+dt,Ci+k2I/2,k2)
        k4R=dt*dCrdt(t+dt,Cr+k3R,Ci+k3I,k1,k1r)
        k4I=dt*dCidt(t+dt,Cr+k3R,Ci+k3I,k1,k1r,k2)
        k4P=dt*dCpdt(t+dt,Ci+k3I,k2)
        Cr=Cr+(k1R+2*(k2R+k3R)+k4R)/6
        Ci=Ci+(k1I+2*(k2I+k3I)+k4I)/6
        Cp=Cp+(k1P+2*(k2P+k3P)+k4P)/6
        t=t+dt;

    C=cP*V*Ci+cR*V*Cr0+cO*tf+cT*tf*(T-298)
    return C#,Ci,Cr

max_stars_repo_name: zavalab/bayesianopt

max_issues_repo_name: zavalab/bayesianopt

max_forks_repo_name: zavalab/bayesianopt

reason: from numpy


text:
# !/usr/bin python3
# encoding    : utf-8 -*-
# @author     : Zijiang Yang
# @file       : Burgers_equation_pinn.py
# @Time       : 2021/12/6 22:04
import os
import sys
sys.path.append(os.curdir)

import time
import yaml
import random
import logging
import numpy as np

import torch
from torch import optim

from models.PINN_models import PINN
from data.DataCreator import BurgersEquationDataCreator

# ---------------------- logger ----------------------
logging.basicConfig(level=logging.NOTSET,
                    format="%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
# create logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# create file handler
run_time = time.strftime('%Y%m%d%H%M', time.localtime(time.time()))
log_file_path = os.path.join("./logs", "{}_{}.log".format(run_time, "burgers_equation"))
fh = logging.FileHandler(log_file_path, mode="w")
fh.setLevel(logging.NOTSET)

# output format
basic_format = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
fh.setFormatter(basic_format)

# add logger to handler
logger.addHandler(fh)

class BurgersEquationPINN(object):

    def __init__(self, conf, load_weight_path=None):
        super(BurgersEquationPINN, self).__init__()

        logger.info("PINN for Burgers Equation \n \n \n")
        logger.info("hyps list {}".format(conf))
        self.conf = conf

        # ------------------------------------------ #
        # ----------------- global ----------------- #
        # ------------------------------------------ #
        # seed
        random.seed(conf["seed"])
        torch.manual_seed(conf["seed"])
        torch.random.manual_seed(conf["seed"])
        np.random.seed(conf["seed"])

        # device
        if torch.cuda.is_available():
            self.device = torch.device("cuda")
        else:
            self.device = torch.device("cpu")

        # ---------------------------------------- #
        # ----------------- data ----------------- #
        # ---------------------------------------- #

        # create numerical data
        self.data_creator = BurgersEquationDataCreator(conf)
        begin_fd_time = time.time()
        logger.info("create data ...")
        self.data_creator.iter_func()
        over_fd_time = time.time()
        logger.info(
            "finite difference with fourth lunge-kutta method uses {:.5f}s".format(over_fd_time - begin_fd_time)
        )
        # plot numerical data
        logger.info("create figure and gif ...")
        self.data_creator.plot_func(save_figure=True)
        logger.info("save figure as {}".format(self.data_creator.figure_output_path))
        logger.info("save gif as {}".format(self.data_creator.gif_output_path))

        # sampling
        data_dict = self.data_creator.sampling(boundary_num=conf["boundary_num"],
                                               initial_num=conf["initial_num"],
                                               common_num=conf["common_num"], seed=conf["seed"])

        # raw data
        boundary_data = data_dict["boundary"]
        initial_data = data_dict["initial"]
        common_data = data_dict["common"]

        u_train_x = np.vstack([initial_data[:, 0:2], boundary_data[:, 0:2]])
        u_train_exact = np.vstack([initial_data[:, 2].reshape((-1, 1)), boundary_data[:, 2].reshape((-1, 1))])
        f_train_x = common_data[:, 0:2]

        self.u_x = torch.tensor(u_train_x[:, 0:1], requires_grad=True).float().to(self.device)
        self.u_t = torch.tensor(u_train_x[:, 1:2], requires_grad=True).float().to(self.device)
        self.f_x = torch.tensor(f_train_x[:, 0:1], requires_grad=True).float().to(self.device)
        self.f_t = torch.tensor(f_train_x[:, 1:2], requires_grad=True).float().to(self.device)
        self.u_exact = torch.tensor(u_train_exact).float().to(self.device)

        # ----------------------------------------- #
        # ----------------- model ----------------- #
        # ----------------------------------------- #
        layers = [20, 20, 20, 20, 20, 20, 20, 20]
        self.pinn_model = PINN(input_dim=2, output_dim=1, dim_list=layers).to(self.device)
        logger.info("create pinn done ...")
        logger.info(self.pinn_model)
        if conf["load_weight"] == "True":
            logger.info("load weight ...")
            self.pinn_model.load_state_dict(torch.load(load_weight_path))

        # ------------------------------------------------------ #
        # ----------------- optimizer and loss ----------------- #
        # ------------------------------------------------------ #
        self.optimizer = optim.LBFGS(self.pinn_model.parameters(), lr=conf["lr"],
                                     max_iter=conf["max_iter"],
                                     max_eval=conf["max_eval"],
                                     history_size=conf["history_size"],
                                     tolerance_grad=eval(conf["tolerance_grad"]),
                                     tolerance_change=1.0 * np.finfo(float).eps,
                                     line_search_fn=conf["line_search_fn"])
        logger.info("create optimizer and criterion done ...")
        self.iter = 0

        # ---------------------------------------- #
        # ----------------- save ----------------- #
        # ---------------------------------------- #
        self.model_output_path = os.path.join(conf["model_output_root"], conf["model_output_name"])
        self.pinn_val_output_figure = os.path.join(conf["figure_output_root"], conf["pinn_figure_output_name"])

    def train(self):
        logger.info("begin train ...")
        begin_train_time = time.time()

        self.pinn_model.train()
        self.optimizer.step(self.loss_func)

        over_train_time = time.time()
        logger.info("train over ...")
        logger.info("using {:.5f}s".format(self.iter, over_train_time-begin_train_time))

        torch.save(self.pinn_model.state_dict(), self.model_output_path)
        logger.info("save model as {}".format(self.model_output_path))

    def loss_func(self):
        self.optimizer.zero_grad()

        u_pred = self.pinn_model(self.u_x, self.u_t)
        f_pred = self.pinn_model(self.f_x, self.f_t)

        # du_dx
        df_dx = torch.autograd.grad(f_pred, self.f_x,
                                    grad_outputs=torch.ones_like(f_pred),
                                    retain_graph=True, create_graph=True)[0]
        # du_dt
        df_dt = torch.autograd.grad(f_pred, self.f_t,
                                    grad_outputs=torch.ones_like(f_pred),
                                    retain_graph=True, create_graph=True)[0]

        # du_dxx
        df_dxx = torch.autograd.grad(df_dx, self.f_x,
                                     grad_outputs=torch.ones_like(df_dx),
                                     retain_graph=True, create_graph=True)[0]

        pde_output = df_dt + f_pred * df_dx - (0.01 / np.pi) * df_dxx

        loss_u = torch.mean((u_pred - self.u_exact) ** 2)
        loss_f = torch.mean(pde_output ** 2)

        total_loss = loss_f + loss_u

        total_loss.backward()

        self.iter += 1
        if self.iter % 100 == 0:
            logger.info("iter: {}".format(self.iter))
            logger.info("total loss:{:.8e}".format(total_loss.item()))
            logger.info("boundary and initial conditions loss:{:.5e}\t "
                        "pde function loss:{:.5e}".format(loss_u.item(), loss_f.item()))
        return total_loss

    def pred_and_valuation(self):
        logger.info("begin pred ...")
        with torch.no_grad():
            self.pinn_model.eval()

            # pred output init
            true_output_matrix = self.data_creator.result_matrix
            x_position_list = np.linspace(self.data_creator.cal_x_range[0], self.data_creator.cal_x_range[1],
                                          self.data_creator.space_n).reshape((-1, 1))
            t_position_list = np.linspace(0, self.data_creator.time_total, self.data_creator.time_n)

            pred_output_matrix = np.zeros((int(self.data_creator.space_n),
                                           int(self.data_creator.time_n)))

            x_position_tensor = torch.from_numpy(x_position_list).type(torch.float32).to(self.device)
            # pred
            begin_pred_time = time.time()
            for t, time_point in enumerate(t_position_list):

                # create input t tensor
                t_tensor = torch.ones_like(x_position_tensor) * time_point

                pred_ = self.pinn_model(x_position_tensor, t_tensor.to(self.device)).detach().cpu().numpy()

                pred_output_matrix[:, t] = np.reshape(pred_.copy(), (-1))

                if t % 1000 == 0:
                    print(np.max(pred_output_matrix[:, t]), np.min(pred_output_matrix[:, t]))
                    logger.info("[{}/{}] pred done ...".format(t, len(t_position_list)))
            over_pred_time = time.time()
            logger.info("pred done ...")
            logger.info("using {:.5f}s ...".format(over_pred_time - begin_pred_time))

        # valuation
        l1_norm = np.linalg.norm(pred_output_matrix.reshape(-1) - true_output_matrix.reshape(-1), 1)
        l2_norm = np.linalg.norm(pred_output_matrix.reshape(-1) - true_output_matrix.reshape(-1))

        max_error = np.max(np.abs(pred_output_matrix - true_output_matrix))
        min_error = np.min(np.abs(pred_output_matrix - true_output_matrix))
        average_error = np.average(np.abs(pred_output_matrix - true_output_matrix))

        logger.info("l1 norm {:.7f}".format(l1_norm))
        logger.info("l2 norm {:.7f}".format(l2_norm))
        logger.info("max error {:.7f}".format(max_error))
        logger.info("min error {:.7f}".format(min_error))
        logger.info("average error {:.7f}".format(average_error))

        self.data_creator.figure_output_path = os.path.join(self.conf["figure_output_root"],
                                                            self.conf["pinn_figure_output_name"])
        self.data_creator.gif_output_path = os.path.join(self.conf["figure_output_root"],
                                                         self.conf["pinn_gif_output_name"])
        self.data_creator.result_matrix = pred_output_matrix.copy()
        self.data_creator.plot_func(save_figure=True)

if __name__ == "__main__":
    _conf = yaml.load(open("./conf/pinn_burgers_equation.yaml"), Loader=yaml.FullLoader)
    weight_path = r"./output/weights/burgers_equation_pinn.pth"
    main_ = BurgersEquationPINN(_conf, weight_path)
    # main_.train()
    main_.pred_and_valuation()


text:
import scipy.io as sio
import os
from .utils import download_dataset
import numpy as np
import time

_dataset = "svhn"
_baseurl = "http://ufldl.stanford.edu/housenumbers/"
_urls = {
    "train_32x32.mat": "train_32x32.mat",
    "test_32x32.mat": "test_32x32.mat",
}

def load(path=None):
    """Street number classification.

    The `SVHN <http://ufldl.stanford.edu/housenumbers/>`_
    dataset is a real-world
    image dataset for developing machine learning and object
    recognition algorithms with minimal requirement on data
    preprocessing and formatting. It can be seen as similar in flavor
    to MNIST (e.g., the images are of small cropped digits), but
    incorporates an order of magnitude more labeled data (over 600,000
    digit images) and comes from a significantly harder, unsolved,
    real world problem (recognizing digits and numbers in natural
    scene images). SVHN is obtained from house numbers in Google
    Street View images.

    Parameters
    ----------
        path: str (optional)
            default $DATASET_PATH, the path to look for the data and
            where the data will be downloaded if not present

    Returns
    -------

        train_images: array

        train_labels: array

        test_images: array

        test_labels: array

    """
    if path is None:
        path = os.environ["DATASET_PATH"]

    download_dataset(path, _dataset, _urls, _baseurl)

    # Load the dataset (download if necessary) and set
    # the class attributess.
    print("Loading svhn")

    t0 = time.time()

    # Train set
    data = sio.loadmat(path + "svhn/train_32x32.mat")
    train_images = data["X"].transpose([3, 2, 0, 1])
    train_labels = np.squeeze(data["y"]) - 1

    # Test set
    data = sio.loadmat(path + "svhn/test_32x32.mat")
    test_images = data["X"].transpose([3, 2, 0, 1])
    test_labels = np.squeeze(data["y"]) - 1

    print("Dataset svhn loaded in", "{0:.2f}".format(time.time() - t0), "s.")
    dataset = {
        "train_set/images": train_images.astype("float32"),
        "train_set/labels": train_labels.astype("int32"),
        "test_set/images": test_images.astype("float32"),
        "test_set/labels": test_labels.astype("int32"),
    }
    return dataset

max_stars_repo_name: RandallBalestriero/TheanoXLA

max_issues_repo_name: RandallBalestriero/TheanoXLA

max_forks_repo_name: RandallBalestriero/TheanoXLA

reason: import numpy,import scipy