text:
"""
gcode_machine - Copyright (c) 2016 Michael Franzl

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
"""

import re
import logging
import math
import numpy as np

class GcodeMachine:
    """ This implements a simple CNC state machine that can be used
    for simulation and processing of G-code.

    For usage, see README.md.

    Callbacks:

    on_feed_change
    : Emitted when a F keyword is parsed from the G-Code.
    : 1 argument: the feed rate in mm/min

    on_var_undefined
    : Emitted when a variable is to be substituted but no substitution value has been set previously.
    : 1 argument: the key of the undefined variable
    """

    def __init__(self, impos=(0,0,0), ics="G54", cs_offsets={"G54":(0,0,0)}):
        """ Initialization.
        """

        self.logger = logging.getLogger('gerbil')

        ## @var line
        # Holds the current Gcode line
        self.line = ""

        ## @var vars
        # A dict holding values for variable substitution.
        self.vars = {}

        ## @var callback method for parent applications
        self.callback = self._default_callback

        ## @var do_feed_override
        # If set to True, F commands will be replaced with the feed
        # speed set in `request_feed`. If set to False, no feed speed
        # processing will be done.
        self.do_feed_override = False

        ## @var do_fractionize_lines
        # If set to True, linear movements over the threshold of
        # `fract_linear_threshold` will be broken down into small line
        # segments of length `fract_linear_segment_len`. If set to False
        # no processing is done on lines.
        self.do_fractionize_lines = False

        ## @var do_fractionize_arcs
        # If set to True, arcs will be broken up into small line segments.
        # If set to False, no processing is done on arcs.
        self.do_fractionize_arcs = False

        ## @var fract_linear_threshold
        # The threshold for the fractionization of lines.
        self.fract_linear_threshold = 1

        ## @var fract_linear_segment_len
        # The length of the segments of fractionized lines.
        self.fract_linear_segment_len = 0.5

        ## @var spindle_factor
        # Scale S by this value
        self.spindle_factor = 1

        ## @var request_feed
        # If `do_feed_override` is True, a F command will be inserted
        # to the currently active command with this value.
        self.request_feed = None

        ## @var current_feed
        # The current feed rate of the machine
        self.current_feed = None

        ## @var contains_feed
        # True if the current line contains a F command
        self.contains_feed = False

        ## @var current_distance_mode
        # Contains the current distance mode of the machine as string
        self.current_distance_mode = "G90"

        ## @var current_motion_mode
        # Contains the current motion mode of the machine as integer (0 to 3)
        self.current_motion_mode = 0

        ## @var current_plane_mode
        # Contains the current plane mode of the machine as string
        self.current_plane_mode = "G17"

        ## @var cs_offsets
        # Coordinate system offsets. A Python dict with
        # 3-tuples as offsets.
        self.cs_offsets = cs_offsets

        ## @var cs
        # Current coordinate system as string.
        self.cs = ics

        ## @var pos_m
        # Contains the current machine position before execution
        # of the currently set line (the target of the last command)
        self.pos_m = list(impos) # initial machine position

        ## @var pos_w
        # Contains the current working position before execution
        # of the currently set line (the target of the last command)
        self.pos_w = list(np.subtract(self.pos_m, self.cs_offsets[self.cs]))

        ## @var target_m
        # Contains the position target of the currently set command in
        # the machine coordinate system.
        self.target_m = [None, None, None]

        ## @var target_m
        # Contains the position target of the currently set command in
        # the currently seleted coordinate system.
        self.target_w = [None, None, None]

        ## @var offset
        # Contains the offset of the arc center from current position
        self.offset = [0, 0, 0]

        ## @var radius
        # Contains the radius of the current arc
        self.radius = None

        ## @var contains_radius
        # True if the current line contains a R word
        self.contains_radius = False

        ## @var current_spindle_speed
        # Contains the current spindle speed (S word)
        self.current_spindle_speed = None

        ## @var contains_spindle
        # True if the current line contains the S word
        self.contains_spindle = False

        ## @var dist
        # Distance that current line will travel
        self.dist = 0

        ## @var dists
        # Distance that current line will travel, in [x,y,z] directions
        self.dist_xyz = [0, 0, 0]

        ## @var line_is_unsupported_cmd
        # True if current line is not in the whitelist
        self.line_is_unsupported_cmd = False

        self.comment = ""

        self.special_comment_prefix = "_gcm"

        # precompile regular expressions
        self._axes_regexps = []
        self._axes_words = ["X", "Y", "Z"]
        for i in range(0, 3):
            word = self._axes_words[i]
            self._axes_regexps.append(re.compile(".*" + word + "([-.\d]+)"))

        self._offset_regexps = []
        self._offset_words = ["I", "J", "K"]
        for i in range(0, 3):
            word = self._offset_words[i]
            self._offset_regexps.append(re.compile(".*" + word + "([-.\d]+)"))

        self._re_radius = re.compile(".*R([-.\d]+)")

        self._re_use_var = re.compile("#(\d*)")
        self._re_set_var = re.compile("^\s*#(\d+)=([\d.-]+)")

        self._re_feed = re.compile(".*F([.\d]+)")
        self._re_feed_replace = re.compile(r"F[.\d]+")

        self._re_spindle = re.compile(".*S([\d]+)")
        self._re_spindle_replace = re.compile(r"S[.\d]+")

        self._re_motion_mode = re.compile("G([0123])*([^\\d]|$)")
        self._re_distance_mode = re.compile("(G9[01])([^\d]|$)")
        self._re_plane_mode = re.compile("(G1[789])([^\d]|$)")
        self._re_cs = re.compile("(G5[4-9])")

        self._re_comment_paren_convert = re.compile("(.*)\((.*?)\)\s*$")
        self._re_comment_paren_replace = re.compile(r"\(.*?\)")
        self._re_comment_get_comment = re.compile("(.*?)(;.*)")

        self._re_match_cmd_number = re.compile("([GMT])(\d+)")
        self._re_expand_multicommands = re.compile("([GMT])")

        ## @var whitelist_commands
        # Strip line from everything that is not in this list
        self.whitelist_commands = {
            "G": [
                # non-modal commands
                4,  # Dwell
                10, # set coordintate system
                28, # Go to predefined position
                30, # Go to predefined position
                53, # move in machine coordinates
                92, # set coordintate system offset

                # motion modes
                0, # fast linear
                1, # slow linear
                2, # slow arc CW
                3, # slow arc CCW
                38, # probe
                80, # canned cycle stop

                # feed rate modes
                93, # inverse
                94, # normal

                # units
                20, # inch
                21, # mm

                # distance modes
                90, # absulute
                91, # incremental

                # plane select
                17, # XY
                18, # ZY
                19, # YZ

                # tool length offset
                43, # set compensation
                49, # cancel compensation

                # coordinate systems
                54,
                55,
                56,
                57,
                58,
                59,
                ],

            "M": [
                # program flow
                0,
                1,
                2,
                30,

                # coolant
                8,
                9,

                # spindle
                3,
                4,
                5,
                ]
            }

        self.logger.info("Preprocessor Class Initialized")

    def reset(self):
        """
        Reset to initial state.
        """
        self.vars = {}
        self.current_feed = None
        self.current_motion_mode = 0
        self.current_distance_mode = "G90"
        self.callback("on_feed_change", self.current_feed)

    @property
    def position_m(self):
        return list(self.pos_m)

    @position_m.setter
    def position_m(self, pos):
        self.pos_m = list(pos)
        self.pos_w = list(np.subtract(self.pos_m, self.cs_offsets[self.cs]))

    @property
    def position_w(self):
        return list(self.pos_w)

    @position_w.setter
    def position_w(self, pos):
        self.pos_w = list(pos)
        self.pos_m = list(np.add(self.pos_w, self.cs_offsets[self.cs]))

    @property
    def current_cs(self):
        return self.cs

    @current_cs.setter
    def current_cs(self, label):
        self.cs = label
        self.pos_w = list(np.subtract(self.pos_m, self.cs_offsets[self.cs]))

    def set_line(self, line):
        """
        Load a Gcode line into the machine. It will be available in `self.line`.

        @param line
        A string of Gcode.
        """
        self.line = line
        self.transform_comments()

    def split_lines(self):
        """
        Some Gcode generating software spit out 'composite' commands
        like "M3 T2". This really is bad practice.

        This method splits such 'composite' commands up into separate
        commands and returns a list of commands. Machine state is not
        changed.
        """
        commands = re.sub(self._re_expand_multicommands, "\n\g<0>", self.line).strip()
        lines = commands.split("\n")
        lines[0] = lines[0] + self.comment # preserve comment
        return lines

    def strip(self):
        """
        Remove blank spaces and newlines from beginning and end, and remove blank spaces from the middle of the line.
        """
        self.line = self.line.replace(" ", "")
        self.line = self.line.strip()

    def tidy(self):
        """
        Strips G-Code not contained in the whitelist.
        """

        # transform [MG]\d to G\d\d for better parsing
        def format_cmd_number(matchobj):
            cmd = matchobj.group(1)
            cmd_nr = int(matchobj.group(2))
            self.line_is_unsupported_cmd = not (cmd in self.whitelist_commands and cmd_nr in self.whitelist_commands[cmd])
            return "{}{:02d}".format(cmd, cmd_nr)

        self.line = re.sub(self._re_match_cmd_number, format_cmd_number, self.line)

        if self.line_is_unsupported_cmd:
            self.line = ";" + self.line + " ;" + self.special_comment_prefix + ".unsupported"

    def parse_state(self):
        """
        This method...

        * parses motion mode
        * parses distance mode
        * parses plane mode
        * parses feed rate
        * parses spindle speed
        * parses arcs (offsets and radi)
        * calculates travel distances

        ... and updates the machine's state accordingly.
        """

        # parse G0 .. G3 and remember
        m = re.match(self._re_motion_mode, self.line)
        if m: self.current_motion_mode = int(m.group(1))

        # parse G90 and G91 and remember
        m = re.match(self._re_distance_mode, self.line)
        if m: self.current_distance_mode = m.group(1)

        # parse G17, G18 and G19 and remember
        m = re.match(self._re_plane_mode, self.line)
        if m: self.current_plane_mode = m.group(1)

        m = re.match(self._re_cs, self.line)
        if m: self.current_cs = m.group(1)

        # see if current line has F
        m = re.match(self._re_feed, self.line)
        self.contains_feed = True if m else False
        if m: self.feed_in_current_line = float(m.group(1))

        # look for spindle S
        m = re.match(self._re_spindle, self.line)
        self.contains_spindle = True if m else False
        if m: self.current_spindle_speed = int(m.group(1))

        # arc parsing and calculations
        if self.current_motion_mode == 2 or self.current_motion_mode == 3:
            self.offset = [None, None, None]
            for i in range(0, 3):
                # loop over I, J, K offsets
                regexp = self._offset_regexps[i]

                m = re.match(regexp, self.line)
                if m: self.offset[i] = float(m.group(1))

            # parses arcs
            m = re.match(self._re_radius, self.line)
            self.contains_radius = True if m else False
            if m: self.radius = float(m.group(1))

        # calculate distance traveled by this G-Code cmd in xyz
        self.dist_xyz = [0, 0, 0]
        for i in range(0, 3):
            # loop over X, Y, Z axes
            regexp = self._axes_regexps[i]

            m = re.match(regexp, self.line)
            if m:
                if self.current_distance_mode == "G90":
                    # absolute distances
                    self.target_m[i] = self.cs_offsets[self.cs][i] + float(m.group(1))
                    self.target_w[i] = float(m.group(1))

                    # calculate distance
                    self.dist_xyz[i] = self.target_m[i] - self.pos_m[i]
                else:
                    # G91 relative distances
                    self.dist_xyz[i] = float(m.group(1))
                    self.target_m[i] += self.dist_xyz[i]
                    self.target_w[i] += self.dist_xyz[i]
            else:
                # no movement along this axis, stays the same
                self.target_m[i] = self.pos_m[i]
                self.target_w[i] = self.pos_w[i]

        # calculate travelling distance
        self.dist = math.sqrt(self.dist_xyz[0] * self.dist_xyz[0] + self.dist_xyz[1] * self.dist_xyz[1] + self.dist_xyz[2] * self.dist_xyz[2])

    def fractionize(self):
        """
        Breaks lines longer than a certain threshold into shorter segments.

        Also breaks circles into segments.

        Returns a list of command strings. Does not update the machine state.
        """

        result = []

        if self.do_fractionize_lines == True and self.current_motion_mode == 1 and self.dist > self.fract_linear_threshold:
            result = self._fractionize_linear_motion()

        elif self.do_fractionize_arcs == True and (self.current_motion_mode == 2 or self.current_motion_mode == 3):
            result = self._fractionize_circular_motion()

        else:
            # this motion cannot be fractionized
            # return the line as it was passed in
            result = [self.line]

        result[0] = result[0] + self.comment # preserve comment
        return result

    def done(self):
        """
        When all processing/inspecting of a command has been done, call this method.
        This will virtually 'move' the tool of the machine if the current command
        is a motion command.
        """
        if not (self.current_motion_mode == 0 or self.current_motion_mode == 1):
            # only G0 and G1 can stay active without re-specifying
            self.current_motion_mode = None

        # move the 'tool'
        for i in range(0, 3):
            # loop over X, Y, Z axes
            if self.target_m[i] != None: # keep state
                self.pos_m[i] = self.target_m[i]
                self.pos_w[i] = self.target_w[i]

        # re-add comment
        self.line += self.comment

        #print("DONE", self.line, self.pos_m, self.pos_w, self.target)

    def find_vars(self):
        """
        Parses all variables in a G-Code line (#1, #2, etc.) and populates
        the internal `vars` dict with corresponding keys and values
        """

        m = re.match(self._re_set_var, self.line)
        if m:
            key = m.group(1)
            val = str(float(m.group(2))) # get rid of extra zeros
            self.vars[key] = val
            self.line = ";" + self.line

        # find variable usages
        keys = re.findall(self._re_use_var, self.line)
        for key in keys:
            if not key in self.vars: self.vars[key] = None

    def substitute_vars(self):
        """
        Substitute a variable with a value from the `vars` dict.
        """
        keys = re.findall(self._re_use_var, self.line)

        for key in keys:
            val = None
            if key in self.vars:
                val = self.vars[key]

            if val == None:
                self.line = ""
                self.callback("on_var_undefined", key)
                return self.line
            else:
                self.line = self.line.replace("#" + key, str(val))
                self.logger.info("SUBSTITUED VAR #{} -> {}".format(key, val))

    def scale_spindle(self):
        if self.contains_spindle:
            # strip the original S setting
            self.line = re.sub(self._re_spindle_replace, "", self.line).strip()
            self.line += "S{:d}".format(int(self.current_spindle_speed * self.spindle_factor))

    def override_feed(self):
        """
        Call this method to

        * get a callback when the current command contains an F word
        *
        """

        if self.do_feed_override == False and self.contains_feed:
            # Notify parent app of detected feed in current line (useful for UIs)
            if self.current_feed != self.feed_in_current_line:
                self.callback("on_feed_change", self.feed_in_current_line)
            self.current_feed = self.feed_in_current_line

        if self.do_feed_override == True and self.request_feed:

            if self.contains_feed:
                # strip the original F setting
                self.logger.info("STRIPPING FEED: " + self.line)
                self.line = re.sub(self._re_feed_replace, "", self.line).strip()

            if (self.current_feed != self.request_feed):
                self.line += "F{:0.1f}".format(self.request_feed)
                self.current_feed = self.request_feed
                self.logger.info("OVERRIDING FEED: " + str(self.current_feed))
                self.callback("on_feed_change", self.current_feed)

    def transform_comments(self):
        """
        Comments in Gcode can be set with semicolon or parentheses.
        This method transforms parentheses comments to semicolon comments.
        """

        # transform () comment at end of line into semicolon comment
        self.line = re.sub(self._re_comment_paren_convert, "\g<1>;\g<2>", self.line)

        # remove all in-line () comments
        self.line = re.sub(self._re_comment_paren_replace, "", self.line)

        m = re.match(self._re_comment_get_comment, self.line)
        if m:
            self.line = m.group(1)
            self.comment = m.group(2)
        else:
            self.comment = ""

    def _fractionize_circular_motion(self):
        """
        This function is a direct port of Grbl's C code into Python (gcode.c)
        with slight refactoring for Python by Michael Franzl.
        See https://github.com/grbl/grbl

        """

        # implies self.current_motion_mode == 2 or self.current_motion_mode == 3

        if self.current_plane_mode == "G17":
            axis_0 = 0 # X axis
            axis_1 = 1 # Y axis
            axis_linear = 2 # Z axis
        elif self.current_plane_mode == "G18":
            axis_0 = 2 # Z axis
            axis_1 = 0 # X axis
            axis_linear = 1 # Y axis
        elif self.current_plane_mode == "G19":
            axis_0 = 1 # Y axis
            axis_1 = 2 # Z axis
            axis_linear = 0 # X axis

        is_clockwise_arc = True if self.current_motion_mode == 2 else False

        # deltas between target and (current) position
        x = self.target_w[axis_0] - self.pos_w[axis_0]
        y = self.target_w[axis_1] - self.pos_w[axis_1]

        if self.contains_radius:
            # RADIUS MODE
            # R given, no IJK given, self.offset must be calculated

            if tuple(self.target_w) == tuple(self.pos_w):
                self.logger.error("Arc in Radius Mode: Identical start/end {}".format(self.line))
                return [self.line]

            h_x2_div_d = 4.0 * self.radius * self.radius - x * x - y * y;

            if h_x2_div_d < 0:
                self.logger.error("Arc in Radius Mode: Radius error {}".format(self.line))
                return [self.line]

            # Finish computing h_x2_div_d.
            h_x2_div_d = -math.sqrt(h_x2_div_d) / math.sqrt(x * x + y * y);

            if not is_clockwise_arc:
                h_x2_div_d = -h_x2_div_d

            if self.radius < 0:
                h_x2_div_d = -h_x2_div_d;
                self.radius = -self.radius;

            self.offset[axis_0] = 0.5*(x-(y*h_x2_div_d))
            self.offset[axis_1] = 0.5*(y+(x*h_x2_div_d))

        else:
            # CENTER OFFSET MODE, no R given so must be calculated

            if self.offset[axis_0] == None or self.offset[axis_1] == None:
                raise Exception("Arc in Offset Mode: No offsets in plane. {}".format(self.line))
                #self.logger.error("Arc in Offset Mode: No offsets in plane")
                #return [self.line]

            # Arc radius from center to target
            x -= self.offset[axis_0]
            y -= self.offset[axis_1]
            target_r = math.sqrt(x * x + y * y)

            # Compute arc radius for mc_arc. Defined from current location to center.
            self.radius = math.sqrt(self.offset[axis_0] * self.offset[axis_0] + self.offset[axis_1] * self.offset[axis_1])

            # Compute difference between current location and target radii for final error-checks.
            delta_r = math.fabs(target_r - self.radius);
            if delta_r > 0.005:
                if delta_r > 0.5:
                    raise Exception("Arc in Offset Mode: Invalid Target. r={:f} delta_r={:f} {}".format(self.radius, delta_r, self.line))
                    #self.logger.warning("Arc in Offset Mode: Invalid Target. r={:f} delta_r={:f} {}".format(self.radius, delta_r, self.line))
                    #return []
                if delta_r > (0.001 * self.radius):
                    raise Exception("Arc in Offset Mode: Invalid Target. r={:f} delta_r={:f} {}".format(self.radius, delta_r, self.line))
                    #self.logger.warning("Arc in Offset Mode: Invalid Target. r={:f} delta_r={:f} {}".format(self.radius, delta_r, self.line))
                    #return []

        #print(self.pos_m, self.target, self.offset, self.radius, axis_0, axis_1, axis_linear, is_clockwise_arc)

        #print("MCARC", self.line, self.pos_w, self.target_w, self.offset, self.radius, axis_0, axis_1, axis_linear, is_clockwise_arc)

        gcode_list = self._mc_arc(self.pos_w, self.target_w, self.offset, self.radius, axis_0, axis_1, axis_linear, is_clockwise_arc)

        return gcode_list

    def _mc_arc(self, position, target, offset, radius, axis_0, axis_1, axis_linear, is_clockwise_arc):
        """
        This function is a direct port of Grbl's C code into Python (motion_control.c)
        with slight refactoring for Python by Michael Franzl.
        See https://github.com/grbl/grbl
        """

        gcode_list = []
        gcode_list.append(";" + self.special_comment_prefix + ".arc_begin[{}]".format(self.line))

        do_restore_distance_mode = False
        if self.current_distance_mode == "G91":
            # it's bad to concatenate many small floating point segments due to accumulating errors
            # each arc will use G90
            do_restore_distance_mode = True
            gcode_list.append("G90")

        center_axis0 = position[axis_0] + offset[axis_0]
        center_axis1 = position[axis_1] + offset[axis_1]
        # radius vector from center to current location
        r_axis0 = -offset[axis_0]
        r_axis1 = -offset[axis_1]
        # radius vector from target to center
        rt_axis0 = target[axis_0] - center_axis0
        rt_axis1 = target[axis_1] - center_axis1

        angular_travel = math.atan2(r_axis0 * rt_axis1 - r_axis1 * rt_axis0, r_axis0 * rt_axis0 + r_axis1 * rt_axis1)

        arc_tolerance = 0.004
        arc_angular_travel_epsilon = 0.0000005

        if is_clockwise_arc: # Correct atan2 output per direction
            if angular_travel >= -arc_angular_travel_epsilon: angular_travel -= 2*math.pi
        else:
            if angular_travel <= arc_angular_travel_epsilon: angular_travel += 2*math.pi

        segments = math.floor(math.fabs(0.5 * angular_travel * radius) / math.sqrt(arc_tolerance * (2 * radius - arc_tolerance)))

        #print("angular_travel:{:f}, radius:{:f}, arc_tolerance:{:f}, segments:{:d}".format(angular_travel, radius, arc_tolerance, segments))

        words = ["X", "Y", "Z"]
        if segments:
            theta_per_segment = angular_travel / segments
            linear_per_segment = (target[axis_linear] - position[axis_linear]) / segments

            position_last = list(position)
            for i in range(1, segments):
                cos_Ti = math.cos(i * theta_per_segment);
                sin_Ti = math.sin(i * theta_per_segment);
                r_axis0 = -offset[axis_0] * cos_Ti + offset[axis_1] * sin_Ti;
                r_axis1 = -offset[axis_0] * sin_Ti - offset[axis_1] * cos_Ti;

                position[axis_0] = center_axis0 + r_axis0;
                position[axis_1] = center_axis1 + r_axis1;
                position[axis_linear] += linear_per_segment;

                gcodeline = ""
                if i == 1:
                    gcodeline += "G1"

                for a in range(0,3):
                    if position[a] != position_last[a]: # only write changes
                        txt = "{}{:0.3f}".format(words[a], position[a])
                        txt = txt.rstrip("0").rstrip(".")
                        gcodeline += txt
                        position_last[a] = position[a]

                if i == 1:
                    if self.contains_feed: gcodeline += "F{:.1f}".format(self.feed_in_current_line)
                    if self.contains_spindle: gcodeline += "S{:d}".format(self.current_spindle_speed)

                gcode_list.append(gcodeline)

        # make sure we arrive at target
        gcodeline = ""
        if segments <= 1:
            gcodeline += "G1"

        for a in range(0,3):
            if target[a] != position[a]:
                txt = "{}{:0.3f}".format(words[a], target[a])
                txt = txt.rstrip("0").rstrip(".")
                gcodeline += txt

        if segments <= 1:
            # no segments were rendered (very small arc) so we have to put S and F here
            if self.contains_feed: gcodeline += "F{:.1f}".format(self.feed_in_current_line)
            if self.contains_spindle: gcodeline += "S{:d}".format(self.current_spindle_speed)

        gcode_list.append(gcodeline)

        if do_restore_distance_mode == True:
          gcode_list.append(self.current_distance_mode)

        gcode_list.append(";" + self.special_comment_prefix + ".arc_end")

        return gcode_list

    def _fractionize_linear_motion(self):
        gcode_list = []
        gcode_list.append(";" + self.special_comment_prefix + ".line_begin[{}]".format(self.line))

        num_fractions = int(self.dist / self.fract_linear_segment_len)

        for k in range(0, num_fractions):
            # render segments
            txt = ""
            if k == 0:
                txt += "G1"

            for i in range(0, 3):
                # loop over X, Y, Z axes
                segment_length = self.dist_xyz[i] / num_fractions
                coord_rel = (k + 1) * segment_length
                if self.current_distance_mode == "G90":
                    # absolute distances
                    coord_abs = self.pos_w[i] + coord_rel
                    if coord_rel != 0:
                        # only output for changes
                        txt += "{}{:0.3f}".format(self._axes_words[i], coord_abs)
                        txt = txt.rstrip("0").rstrip(".")
                else:
                    # relative distances
                    txt += "{}{:0.3f}".format(self._axes_words[i], segment_length)
                    txt = txt.rstrip("0").rstrip(".")

            if k == 0:
                if self.contains_feed: txt += "F{:.1f}".format(self.feed_in_current_line)
                if self.contains_spindle: txt += "S{:d}".format(self.current_spindle_speed)

            gcode_list.append(txt)

        gcode_list.append(";" + self.special_comment_prefix + ".line_end")
        return gcode_list

    def _default_callback(self, status, *args):
        print("PREPROCESSOR DEFAULT CALLBACK", status, args)

max_stars_repo_name: MagnusR/Gcode-Transformator

max_issues_repo_name: MagnusR/Gcode-Transformator

max_forks_repo_name: MagnusR/Gcode-Transformator


text:
#%% Import Calls
import os
import ogr
import osr
import subprocess
import gdal
import csv
import cv2
import math
import datetime
from pixel import geoUtils
import numpy as np
import random
from guessGenerator import createTranslateMeters

def createBoxFromPoint(xCenter, yCenter, xLength, yLength):

    minX = xCenter - xLength/2
    minY = yCenter - yLength/2
    maxX = xCenter + xLength/2
    maxY = yCenter + yLength/2

    ring = ogr.Geometry(ogr.wkbLinearRing)
    ring.AddPoint(minX, minY)
    ring.AddPoint(minX, maxY)
    ring.AddPoint(maxX, maxY)
    ring.AddPoint(maxX, minY)
    ring.AddPoint(minX, minY)
    polyEnvelope = ogr.Geometry(ogr.wkbPolygon)
    polyEnvelope.AddGeometry(ring)

    return polyEnvelope

def createIoUTest(dstFileName, xCenter=1000, yCenter=1000, xLength=10, yLength=10, translateRange=np.arange(50)):

    srs = osr.SpatialReference()
    srs.ImportFromEPSG(4326)
    # Create the output shapefile
    geoJSONdriver = ogr.GetDriverByName("GeoJson")
    if os.path.exists(dstFileName):
        geoJSONdriver.DeleteDataSource(dstFileName)
    outDataSource = geoJSONdriver.CreateDataSource(dstFileName)
    outLayer = outDataSource.CreateLayer(dstFileName, srs, geom_type=ogr.wkbPolygon)
    # Create the feature and set values

    # add Intersection Field
    idField = ogr.FieldDefn("Intersection", ogr.OFTReal)
    outLayer.CreateField(idField)
    # add Union Field
    idField = ogr.FieldDefn("Union", ogr.OFTReal)
    outLayer.CreateField(idField)
    # add Area Field
    idField = ogr.FieldDefn("Area", ogr.OFTReal)
    outLayer.CreateField(idField)
    # add IOU Field
    idField = ogr.FieldDefn("IOU", ogr.OFTReal)
    outLayer.CreateField(idField)
    # add transX Field
    idField = ogr.FieldDefn("transX", ogr.OFTReal)
    outLayer.CreateField(idField)
    # add transY
    idField = ogr.FieldDefn("transY", ogr.OFTReal)
    outLayer.CreateField(idField)

    featureDefn = outLayer.GetLayerDefn()

    # create InitialPolygon
    polyBaseTruth = createBoxFromPoint(xCenter, yCenter, xLength, yLength)
    feature = ogr.Feature(featureDefn)
    feature.SetGeometry(polyBaseTruth)
    calcIntersection = polyBaseTruth.Intersection(polyBaseTruth).GetArea()
    calcUnion        = polyBaseTruth.Union(polyBaseTruth).GetArea()
    calcArea         = polyBaseTruth.GetArea()
    calcIOU          = calcIntersection/calcUnion
    feature.SetField('Intersection', calcIntersection)
    feature.SetField('Union', calcUnion)
    feature.SetField('Area', calcArea)
    feature.SetField('IOU', calcIOU)
    feature.SetField('transX', 0)
    feature.SetField('transY', 0)
    outLayer.CreateFeature(feature)

    for translateDistanceMetersX in translateRange:
        for translateDistanceMetersY in translateRange:
            print translateDistanceMetersX
            print translateDistanceMetersY
            polyGuess = createTranslateMeters(polyBaseTruth, translateDistanceMetersX,
                          translateDistanceMetersY, wgs84Input=False)
            print polyGuess.ExportToJson()
            feature = ogr.Feature(featureDefn)
            feature.SetGeometry(polyGuess)
            calcIntersection = polyBaseTruth.Intersection(polyGuess).GetArea()
            calcUnion = polyBaseTruth.Union(polyGuess).GetArea()
            calcArea = polyGuess.GetArea()
            calcIOU = calcIntersection / calcUnion
            feature.SetField('Intersection', calcIntersection)
            feature.SetField('Union', calcUnion)
            feature.SetField('Area', calcArea)
            feature.SetField('IOU', calcIOU)
            feature.SetField('transX', translateDistanceMetersX)
            feature.SetField('transY', translateDistanceMetersY)

            outLayer.CreateFeature(feature)

    outDataSource.Destroy()

if __name__ == '__main__':
    dstFileName = '/Users/dlindenbaum/Documents/CosmiQCode_09282015/BEE-CSharp/data/Rio_Submission_Testing_CQW/iouTest_50x50.geojson'
    createIoUTest(dstFileName, xCenter=1000, yCenter=1000, xLength=50, yLength=50, translateRange=np.arange(50))

ext: py

max_stars_repo_name: toddstavish/BEE-CSharp

max_issues_repo_name: toddstavish/BEE-CSharp

max_forks_repo_name: toddstavish/BEE-CSharp

reason: import numpy


text:
"""This module implements the Link class.
"""

import numpy as np
import sympy

from pyik import geometry_utils

class Link(object):
    """Base Link class.

    :param name: The name of the link
    :type name: string
    :param bounds: Optional : The bounds of the link. Defaults to None
    :type bounds: tuple
    :param use_symbolic_matrix: wether the transformation matrix is stored as
        Numpy array or as a Sympy symbolic matrix.
    :type use_symbolic_matrix: bool
    """

    def __init__(self, name, bounds=(None, None)):
        self.bounds = bounds
        self.name = name

    def _get_rotation_axis(self):
        # Defaults to None
        return [0, 0, 0, 1]

    def get_transformation_matrix(self, theta):
        raise NotImplementedError

class OriginLink(Link):
    """The link at the origin of the robot
    """
    def __init__(self):
        Link.__init__(self, name='Base link')
        self._length = 1

    def _get_rotation_axis(self):
        return [0, 0, 0, 1]

    def get_transformation_matrix(self, theta):
        return np.eye(4)

class URDFLink(Link):
    """Link in URDF representation.

    :param name: The name of the link
    :type name: string
    :param bounds: Optional : The bounds of the link. Defaults to None
    :type bounds: tuple
    :param translation: The translation vector. (In URDF, attribute "xyz"
        of the "origin" element).
    :type translation: numpy.array
    :param orientation: The orientation of the link. (In URDF, attribute "rpy"
        of the "origin" element).
    :type orientation: numpy.array
    :param rotation: The rotation axis of the link. (In URDF, attribute "xyz" of
        the "axis" element).
    :type rotation: numpy.array
    :param angle_representation: Optionnal : The representation used by the
    angle. Currently supported representations : rpy. Defaults to rpy, the URDF
        standard.
    :type angle_representation: string
    :param use_symbolic_matrix: wether the transformation matrix is stored as a
        Numpy array or as a Sympy symbolic matrix.
    :type use_symbolic_matrix: bool
    :returns: The link object
    :rtype: URDFLink
    :Example:

    URDFlink()
    """

    def __init__(self,
                 name,
                 translation,
                 orientation,
                 rotation,
                 bounds=(None, None),
                 angle_representation='rpy',
                 use_symbolic_matrix=True):

        Link.__init__(self, name=name, bounds=bounds)
        self.use_symbolic_matrix = use_symbolic_matrix
        self.translation = np.array(translation)
        self.orientation = np.array(orientation)
        self.rotation = np.array(rotation)

        self._length = np.linalg.norm(translation)
        self._axis_length = self._length

        if use_symbolic_matrix:
            theta = sympy.symbols('theta')

            symbolic_frame_matrix = np.eye(4)

            # Apply translation matrix
            symbolic_frame_matrix = symbolic_frame_matrix * sympy.Matrix(
                    geometry_utils.homogeneous_translation_matrix(*translation))

            # Apply orientation matrix
            symbolic_frame_matrix = (
                    symbolic_frame_matrix *
                    geometry_utils.cartesian_to_homogeneous(
                        geometry_utils.rpy_matrix(*orientation))
                    )

            # Apply rotation matrix
            symbolic_frame_matrix = (
                symbolic_frame_matrix *
                geometry_utils.cartesian_to_homogeneous(
                    geometry_utils.symbolic_axis_rotation_matrix(
                        rotation, theta), matrix_type='sympy')
                )

            self.symbolic_transformation_matrix = sympy.lambdify(
                    theta, symbolic_frame_matrix, 'numpy')

    def _get_rotation_axis(self):
        return np.dot(
                geometry_utils.homogeneous_translation_matrix(
                        *self.translation),
                np.dot(geometry_utils.cartesian_to_homogeneous(
                           geometry_utils.rpy_matrix(*self.orientation)),
                       geometry_utils.cartesian_to_homogeneous_vectors(
                           self.rotation * self._axis_length))
                )

    def get_transformation_matrix(self, theta):
        if self.use_symbolic_matrix:
            frame_matrix = self.symbolic_transformation_matrix(theta)
        else:
            # Init the transformation matrix
            frame_matrix = np.eye(4)

            # First, apply translation matrix
            frame_matrix = np.dot(
                    frame_matrix,
                    geometry_utils.homogeneous_translation_matrix(
                        *self.translation)
                    )

            # Apply orientation
            frame_matrix = np.dot(
                    frame_matrix,
                    geometry_utils.cartesian_to_homogeneous(
                        geometry_utils.rpy_matrix(*self.orientation))
                    )

            # Apply rotation matrix
            frame_matrix = np.dot(
                    frame_matrix,
                    geometry_utils.cartesian_to_homogeneous(
                        geometry_utils.axis_rotation_matrix(
                            self.rotation, theta))
                    )

        return frame_matrix

max_stars_repo_name: kuanfang/pyik

max_issues_repo_name: kuanfang/pyik

max_forks_repo_name: kuanfang/pyik

reason: import numpy,import sympy


text:
#!/usr/bin/env python

"""Tools to convert raw DOLPHOT output to other formats.

May be executed from the command line.
EXTREMELY WORK IN PROGRESS.

"""

import pandas as pd
import re
import vaex

from astropy.io import fits
from astropy.wcs import WCS

__all__ = ['read_colfile', 'ascii_to_vaex', 'select_cols']

# Mappings of DOLPHOT column descriptions to column names
replace_single = {
    '((Total)|(Measured)) counts'       : 'COUNT',
    '((Total)|(Measured)) sky level'    : 'SKY',
    'Normalized count rate uncertainty' : 'RATERR',
    'Normalized count rate'             : 'RATE',
    'Instrumental VEGAMAG magnitude'    : 'VEGA',
    'Transformed UBVRI magnitude'       : 'TRANS',
    'Magnitude uncertainty'             : 'ERR',
    'Photometry quality flag'           : 'FLAG',
}

replace_global = {
    'Signal-to-noise' : 'SNR',
    '(ness)|(ing)'    : '',
}

def read_colfile(colfile, replace_single=replace_single,
                 replace_global=replace_global):
    """Construct a table of column names for DOLPHOT output.

    Inputs
    ------
    colfile : str, path object or file-like object
        Path to DOLPHOT '.column' file; see pandas.read_csv

    Returns
    -------
    df_col : pandas.DataFrame
        A table of column descriptions and their corresponding names.
    """
    # read column file, splitting into index (column number) and description
    df_col = pd.read_csv(colfile, sep=re.escape('. '), names=['desc'],
                         index_col=0, engine='python')
    # split into "single" and "global" based on having a ', ' before ' ('
    is_single = df_col['desc'].str.split(re.escape(' (')).str[0].\
        str.contains(re.escape(', '))
    df_single = df_col.loc[is_single]
    df_global = df_col.drop(df_single.index)

    # get rid of 'Object' and select first word
    names_global = df_global.desc.str.replace('Object', '').\
        str.strip().str.split().str[0]

    single_split = df_single['desc'].str.split(re.escape(', '))

    names_single = single_split.str[0]
    for k, v in replace_single.items():
        names_single = names_single.replace(re.compile(k), v)

    names_concat = pd.concat([names_global, names_single]
                             ).reindex_like(df_col)
    for k, v in replace_global.items():
        names_concat = names_concat.replace(re.compile(k), v)

    prefix_global = pd.Series('', index=names_global.index)
    prefix_single = single_split.str[1].str.split().str[0].\
        str.replace(re.escape('.chip'), '_chip')
    prefix_concat = pd.concat([prefix_global, prefix_single]
                              ).reindex_like(df_col)
    names = prefix_concat.str.cat(names_concat.str.upper(), sep='_'
                                  ).str.replace('^_', '')
    df_col.loc[:, 'names'] = names
    return df_col

def ascii_to_vaex(asciifile, df_col=None, names=None, usecols=None, **kwargs):
    """Read raw DOLPHOT output photometry into Vaex.

    Inputs
    ------
    asciifile : str, path object or file-like object
        Path to DOLPHOT photometry file; see `pandas.read_csv`
    df_col : pandas.DataFrame, optional
        DataFrame read in from DOLPHOT columns file; see `read_colfile`
    names : list-like, optional
        Sequence of column names; see `pandas.read_csv`
        Either `names` or `df_col` must be specified.
    usecols : list-like or callable, optional
        Subset of columns to be read in; see `pandas.read_csv`
        If None, assumes `names` corresponds to first N columns
        Default: None
    **kwargs
        Other keyword arguments to be passed to `vaex.from_csv`

    Returns
    -------
    ds : vaex.dataframe.DataFrame
        Photometry table.
    """
    if df_col is not None:
        names = df_col['names'].tolist()
        usecols = (df_col.index - 1).tolist()
    else:
        usecols = list(range(len(names))) if usecols is None else usecols
    # null mag values are 99.999, null mag err values are 9.999
    na_values = {n : '99.999' for n in names if
                 (n.endswith('VEGA') | n.endswith('TRANS'))}
    na_values.update({n : '9.999' for n in names if n.endswith('ERR')})
    ds = vaex.from_csv(asciifile, names=names, usecols=usecols, header=None,
                       delim_whitespace=True, na_values=na_values,
                       float_precision='round_trip', **kwargs)
    return ds

# for m31: copy_index=True, chunk_size=100_000, convert=gscratch/scrubbed
# HERE BE DRAGONS aka non-generalized functions

def select_cols(df_col, regex='(^[X,Y]$)|(^[F,G]Q?[0-9]{2,5}[W,M,N,X,L]P?_)'):
    """Select columns to be used in ascii_to_vaex.

    Only for M33 use right now!!!
    Regex: X + Y + all columns beginning with any ACS/WFC3 filter name
    """
    cols = df_col.loc[df_col.names.str.match(regex), 'names']
    names = cols.tolist()
    usecols = (cols.index - 1).tolist()
    return names, usecols

def conv_pix_wcs(x, y, wcsfile):
    # dolphot coordinates are half a pixel off from everything else
    # i yearn for death
    w = WCS(fits.Header.fromtextfile(wcsfile))
    return w.all_pix2world(x, y, 0.5)

def add_wcs(ds, base, firstcols=['RA', 'DEC', 'X', 'Y']):
    if 'RA' in ds.get_column_names():
        ds.drop('RA', inplace=True, check=False)
    if 'DEC' in ds.get_column_names():
        ds.drop('DEC', inplace=True, check=False)
    wcsfile = f'{base}/{base}_F475W_drc_wcs.txt'
    ra, dec = conv_pix_wcs(*ds.evaluate(['X', 'Y']), wcsfile)
    ds.add_column('RA', ra)
    ds.add_column('DEC', dec)
    names = firstcols + [n for n in ds.get_column_names()
                         if n not in firstcols]
    return ds[names]

def calc_xmed(ds_left, ds_right):
    # get center pixel for overlapping photometry
    return int(round((ds_right.X.min() + ds_left.X.max()) / 2))

def merge_parallel_phot(name, export=True):
    # merge 3 photsec catalogs
    ds1 = vaex.open(f'{name}/{name}_1.phot.hdf5')
    ds2 = vaex.open(f'{name}/{name}_2.phot.hdf5')
    ds3 = vaex.open(f'{name}/{name}_3.phot.hdf5')
    x0, x1 = calc_xmed(ds1, ds2), calc_xmed(ds2, ds3)
    ds1.select(f'X <= {x0}', name='__filter__')
    ds2.select_box(['X'], [[x0, x1]], name='__filter__')
    ds3.select(f'X >= {x1}', name='__filter__')
    ds = vaex.dataframe.DataFrameConcatenated([d.extract() for d in
                                               [ds1, ds2, ds3]])
    ds = add_wcs(ds, name)
    if export:
        ds.export_hdf5(f'{name}/{name}.phot.hdf5')
        print(f'Merged catalogs written to {name}/{name}.phot.hdf5')
    else:
        return ds

if __name__ == '__main__':
    import sys
    import os
    base = sys.argv[1]
    os.system(f'gzip {base}/*.phot')
    for i in range(1, 4):
        photfile = f'{base}/{base}_{i}.phot'
        colfile = photfile + '.columns'
        hdffile = photfile + '.hdf5'
        df_col = read_colfile(colfile)
        names, usecols = select_cols(df_col)
        ascii_to_vaex(photfile + '.gz', names=names,
                      usecols=usecols).export_hdf5(hdffile)
        print(f'Written {hdffile}')
    merge_parallel_phot(base)

max_stars_repo_name: meredith-durbin/remora

max_issues_repo_name: meredith-durbin/remora

max_forks_repo_name: meredith-durbin/remora

reason: from astropy


text:
# Copyright (c) 2020 Graphcore Ltd. All rights reserved.
import numpy as np
import pytest
import popart
import pprint
import json
import onnx
from onnx import numpy_helper

# `import test_util` requires adding to sys.path
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).resolve().parent.parent))
import test_util as tu

def test_dropout_outlinable():
    N = 20
    K = 40
    layers = 6
    dropoutRatio = 0.7

    def model():
        np.random.seed(253)
        input_data = np.random.rand(1, N, K).astype(np.float32)
        weight_data = np.random.rand(1, K, K).astype(np.float32)

        builder = popart.Builder()
        d0 = builder.addInputTensor(popart.TensorInfo('FLOAT', (1, N, K)),
                                    'data0')
        x = d0

        for i in range(layers):
            w = builder.addInitializedInputTensor(weight_data,
                                                  debugContext='weight_%d' % i)
            x = builder.aiOnnx.matmul([x, w], debugContext='matmul_%d' % i)
            [x] = builder.aiOnnx.dropout([x],
                                         debugContext='dropout_%d' % i,
                                         num_outputs=1,
                                         ratio=dropoutRatio)

        loss = builder.aiGraphcore.l1loss([x], 0.1, debugContext='loss')
        return builder.getModelProto(), {d0: input_data}, x, loss

    def run_test():
        proto, data, x, loss = model()
        options = popart.SessionOptions()

        dataFlow = popart.DataFlow(1, {x: popart.AnchorReturnType("ALL")})
        with tu.create_test_device() as device:
            session = popart.TrainingSession(fnModel=proto,
                                             dataFlow=dataFlow,
                                             deviceInfo=device,
                                             userOptions=options,
                                             loss=loss,
                                             optimizer=popart.ConstSGD(0.01))

            session.prepareDevice()
            session.weightsFromHost()
            anchors = session.initAnchorArrays()
            stepio = popart.PyStepIO(data, anchors)
            session.run(stepio)
            ir = json.loads(
                session._serializeIr(popart.IrSerializationFormat.JSON))
            dropouts = [
                op for op in ir['maingraph'] if op['type'] == 'Dropout'
            ]
            assert (len(dropouts) <= 2)
            device.detach()
            return anchors[x]

    outputs_run1 = run_test()


text:
# To find Most visited URL categories

import pandas as pd
from pprint import pprint
from collections import Counter
from ast import literal_eval
import os
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm

file = 'C:/Users/hkuad/Desktop/Subjects/DA/DataSets/NewData/DataSet2/http_info.csv'

url_category = '../url-sentiment-analysis/url_category_hits_count.csv'

def generate_url_category_hits():
    http_df = pd.read_csv(file, usecols=[4])

    url_df = pd.read_csv('../url-sentiment-analysis/url_google_sentiment_analysis.csv', usecols=[0, 5])

    url_df['category'] = url_df['category'].apply(literal_eval)

    url_df = pd.merge(http_df, url_df, on='url')['category']

    print(url_df)

    c = Counter()

    for row in url_df:
        c.update(row)

    pprint(c)

    df = pd.DataFrame.from_dict(c, orient='index').reset_index()

    df = df.rename(columns={'index': 'category', 0: 'count'})

    df.sort_values('count', ascending=False, inplace=True)

    df.reset_index(inplace=True, drop=True)

    df.to_csv(url_category, index=False)

    return df

if os.path.exists(url_category):
    url_cat_df = pd.read_csv(url_category, sep=',')
else:
    url_cat_df = generate_url_category_hits()

print(url_cat_df)

cat_dict = {'Food & Drink': 0, 'Beauty & Fitness': 0, 'Law & Government': 0, 'Sports': 0, 'Reference': 0,
            'Real Estate': 0, 'Games': 0, 'Science': 0, 'Hobbies & Leisure': 0, 'Business & Industrial': 0,
            'Autos & Vehicles': 0, 'Travel': 0, 'Home & Garden': 0, 'Shopping': 0, 'News': 0, 'Internet & Telecom': 0,
            'Online Communities': 0, 'People & Society': 0, 'Finance': 0, 'Pets & Animals': 0, 'Health': 0,
            'Books & Literature': 0, 'Computers & Electronics': 0, 'Arts & Entertainment': 0, 'Jobs & Education': 0,
            'Adult': 0}

for i in range(0, url_cat_df.shape[0]):
    for key in cat_dict.keys():
        if key in url_cat_df['category'][i]:
            cat_dict[key] += url_cat_df['count'][i]
            print(key, ' ', url_cat_df['category'][i])

pprint(cat_dict)

cat_dict = dict((k, v) for k, v in cat_dict.items() if v > 0)

pprint(cat_dict)

plt.gca().axis("equal")
labels = cat_dict.keys()

pie = plt.pie([float(v) for v in cat_dict.values()],
              startangle=0)

plt.legend(pie[0], labels, bbox_to_anchor=(1, 0.5),
           loc="center right", fontsize=10,
           bbox_transform=plt.gcf().transFigure)

plt.subplots_adjust(left=0.0, bottom=0.1, right=0.7)

plt.pie([float(v) for v in cat_dict.values()], labels=cat_dict.keys(), autopct=None)

plt.show()

plt.figure(0)

plt.bar(range(len(cat_dict)), cat_dict.values(), align='center')
plt.xticks(range(len(cat_dict)), cat_dict.keys(), rotation=90)
plt.gcf().subplots_adjust(bottom=0.3)

plt.show()


text:
import importlib
import json
import numpy as np
import os
import re
import sys
import time
import uuid
import copy

from hypergan.discriminators import *
from hypergan.distributions import *
from hypergan.generators import *
from hypergan.inputs import *
from hypergan.samplers import *
from hypergan.trainers import *

import hyperchamber as hc
from hyperchamber import Config
from hypergan.ops import TensorflowOps
import hypergan as hg

from hypergan.gan_component import ValidationException, GANComponent
from .base_gan import BaseGAN

class ConfigurableGAN(BaseGAN):
    def __init__(self, *args, **kwargs):
        self.d_terms = []
        self.Ds = []
        BaseGAN.__init__(self, *args, **kwargs)

    def create_encoder(self):
        return self.create_component(self.config.encoder)

    def create_latent(self, zi):
        return self.create_component(self.config.latent)

    def create_generator(self):
        return self.create_component(self.config.generator)

    def parse_opts(self, opts):
        options = {}
        for opt in opts.split(","):
            if opt == "":
                continue
            name, val = opt.split("=")
            value = self.configurable_param(val)
            options[name]=value
        return hc.Config(options)

    def required(self):
        return "terms".split()

    def create_term(self, term):
        for match in
        matching = {
                "gN(eN(xN))": self.create_generator,
                "gN(zN)": self.create_generator
                }
        for regex, method in matching.items():
            regex_subbed = regex.replace("(", '\(').replace(")", '\)').replace("N", "(\d+)?").replace(",options", "([-,=\w\d\.\(\)]+)?")
            regex = re.compile(regex_subbed)
            args = re.match(regex, term)
            if args:
                return method(*args.groups())

        raise ValidationException("Could not match term: " + term)

    def forward_term(self, term):
        matching = {
                "gN(eN(xN))": self.geN,
                "gN(zN)": self.gzN,
                "xN": self.xN
                }
        for regex, method in matching.items():
            regex_subbed = regex.replace("(", '\(').replace(")", '\)').replace("N", "(\d+)?").replace(",options", "([-,=\w\d\.\(\)]+)?")
            regex = re.compile(regex_subbed)
            args = re.match(regex, term)
            if args:
                return method(*args.groups())

        raise ValidationException("Could not match term: " + term)

    def create(self):
        config = self.config

        self.latent = hc.Config({"sample": self.zN(0)})
        self.discriminators = []
        self.losses = []
        for i,term in enumerate(self.config.terms):
            dN, args = term.split(":")

            d_terms = args.split("/")
            terms = []
            for dt in d_terms:
                terms += (term,self.create_term(dt))
            reuse = False

            dN = re.findall("\d+", dN)[0]
            dN = int(dN)
            tfname = "d"+str(dN)
            D = self.create_component(config.discriminator)
            self.Ds.append(D)
            self.d_terms += terms

        self.trainer = self.create_component(config.trainer)

    def create_controls(self, z_shape):
        direction = tf.constant(0.0, shape=z_shape, name='direction') * 1.00
        slider = tf.constant(0.0, name='slider', dtype=tf.float32) * 1.00
        return direction, slider

    def forward_pass(self):
        d_reals = []
        d_fakes = []
        for terms in d_terms:

        return d_reals, d_fakes

    def forward_loss(self):
        losses = []
        for d_real, d_fake in zip(d_reals, d_fakes):
            loss = self.create_component(config.loss, discriminator=d, split=len(d_terms))
            d_loss, g_loss = loss.forward(d_real, d_fake)
            d_loss = [self.configurable_param(config.term_gammas[i]) * d_loss, self.configurable_param(config.term_gammas[i]) * g_loss]
            losses += [[d_loss, g_loss]]

        self.loss = hc.Config({
            'sample': [sum([l.sample[0] for l in losses]), sum([l.sample[1] for l in losses])]
            })

    def g_parameters(self):
        params = []
        for d_terms in self.d_terms:
            for term in d_terms:
                params += term[1].parameters()
        return params

    def d_parameters(self):
        params = []
        for m in self.Ds:
            params += m.parameters()
        return params

max_stars_repo_name: limberc/HyperGAN

max_issues_repo_name: limberc/HyperGAN

max_forks_repo_name: limberc/HyperGAN


text:
import io
import os
import zipfile
from urllib.request import urlretrieve

import h5py
import numpy as np
import requests
from matplotlib.pyplot import imread, imsave
from scipy.io import loadmat
from scipy.ndimage import gaussian_filter

def download_salicon(data_path):
    """Downloads the SALICON dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.

    .. seealso:: The code for downloading files from google drive is based
                 on the solution provided at [https://bit.ly/2JSVgMQ].
    """

    print(">> Downloading SALICON dataset...", end="", flush=True)

    default_path = data_path + "salicon/"
    fixations_path = default_path + "fixations/"
    saliency_path = default_path + "saliency/"

    os.makedirs(fixations_path, exist_ok=True)
    os.makedirs(saliency_path, exist_ok=True)

    ids = ["1g8j-hTT-51IG1UFwP0xTGhLdgIUCW5e5",
           "0B2hsWbciDVedS1lBZHprdXFoZkU",
           "0B2hsWbciDVedNWJZMlRxeW1PY1U"]

    urls = ["https://drive.google.com/uc?id=" +
            i + "&export=download" for i in ids]

    save_paths = [default_path, fixations_path, saliency_path]

    session = requests.Session()

    for count, url in enumerate(urls):
        response = session.get(url, params={"id": id}, stream=True)
        token = _get_confirm_token(response)

        if token:
            params = {"id": id, "confirm": token}
            response = session.get(url, params=params, stream=True)

        _save_response_content(response, data_path + "tmp.zip")

        with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
            for file in zip_ref.namelist():
                if "test" not in file:
                    zip_ref.extract(file, save_paths[count])

    os.rename(default_path + "images", default_path + "stimuli")

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_mit1003(data_path):
    """Downloads the MIT1003 dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.
    """

    print(">> Downloading MIT1003 dataset...", end="", flush=True)

    default_path = data_path + "mit1003/"
    stimuli_path = default_path + "stimuli/"
    fixations_path = default_path + "fixations/"
    saliency_path = default_path + "saliency/"

    os.makedirs(stimuli_path, exist_ok=True)
    os.makedirs(fixations_path, exist_ok=True)
    os.makedirs(saliency_path, exist_ok=True)

    url = "https://people.csail.mit.edu/tjudd/WherePeopleLook/ALLSTIMULI.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            if file.endswith(".jpeg"):
                file_name = os.path.split(file)[1]
                file_path = stimuli_path + file_name

                with open(file_path, "wb") as stimulus:
                    stimulus.write(zip_ref.read(file))

    url = "https://people.csail.mit.edu/tjudd/WherePeopleLook/ALLFIXATIONMAPS.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            file_name = os.path.split(file)[1]

            if file.endswith("Pts.jpg"):
                file_path = fixations_path + file_name

                # this file is mistakenly included in the dataset and can be ignored
                if file_name == "i05june05_static_street_boston_p1010764fixPts.jpg":
                    continue

                with open(file_path, "wb") as fixations:
                    fixations.write(zip_ref.read(file))

            elif file.endswith("Map.jpg"):
                file_path = saliency_path + file_name

                with open(file_path, "wb") as saliency:
                    saliency.write(zip_ref.read(file))

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_cat2000(data_path):
    """Downloads the CAT2000 dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.
    """

    print(">> Downloading CAT2000 dataset...", end="", flush=True)

    default_path = data_path + "cat2000/"

    os.makedirs(data_path, exist_ok=True)

    url = "http://saliency.mit.edu/trainSet.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            if not("Output" in file or "allFixData" in file):
                zip_ref.extract(file, data_path)

    os.rename(data_path + "trainSet/", default_path)

    os.rename(default_path + "Stimuli", default_path + "stimuli")
    os.rename(default_path + "FIXATIONLOCS", default_path + "fixations")
    os.rename(default_path + "FIXATIONMAPS", default_path + "saliency")

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_dutomron(data_path):
    """Downloads the DUT-OMRON dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.
    """

    print(">> Downloading DUTOMRON dataset...", end="", flush=True)

    default_path = data_path + "dutomron/"
    stimuli_path = default_path + "stimuli/"
    fixations_path = default_path + "fixations/"
    saliency_path = default_path + "saliency/"

    os.makedirs(stimuli_path, exist_ok=True)
    os.makedirs(fixations_path, exist_ok=True)
    os.makedirs(saliency_path, exist_ok=True)

    url = "http://saliencydetection.net/dut-omron/download/DUT-OMRON-image.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            if file.endswith(".jpg") and not "._" in file:
                file_name = os.path.basename(file)
                file_path = stimuli_path + file_name

                with open(file_path, "wb") as stimulus:
                    stimulus.write(zip_ref.read(file))

    url = "http://saliencydetection.net/dut-omron/download/DUT-OMRON-eye-fixations.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            if file.endswith(".mat") and not "._" in file:
                file_name = os.path.basename(file)
                file_name = os.path.splitext(file_name)[0] + ".png"

                loaded_zip = io.BytesIO(zip_ref.read(file))

                fixations = loadmat(loaded_zip)["s"]
                sorted_idx = fixations[:, 2].argsort()
                fixations = fixations[sorted_idx]

                size = fixations[0, :2]

                fixations_map = np.zeros((size[1], size[0]))

                fixations_map[fixations[1:, 1],
                              fixations[1:, 0]] = 1

                saliency_map = gaussian_filter(fixations_map, 16)

                imsave(saliency_path + file_name, saliency_map, cmap="gray")
                imsave(fixations_path + file_name, fixations_map, cmap="gray")

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_pascals(data_path):
    """Downloads the PASCAL-S dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.
    """

    print(">> Downloading PASCALS dataset...", end="", flush=True)

    default_path = data_path + "pascals/"
    stimuli_path = default_path + "stimuli/"
    fixations_path = default_path + "fixations/"
    saliency_path = default_path + "saliency/"

    os.makedirs(stimuli_path, exist_ok=True)
    os.makedirs(fixations_path, exist_ok=True)
    os.makedirs(saliency_path, exist_ok=True)

    url = "http://cbs.ic.gatech.edu/salobj/download/salObj.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            file_name = os.path.basename(file)

            if file.endswith(".jpg") and "imgs/pascal" in file:
                file_path = stimuli_path + file_name

                with open(file_path, "wb") as stimulus:
                    stimulus.write(zip_ref.read(file))

            elif file.endswith(".png") and "pascal/humanFix" in file:
                file_path = saliency_path + file_name

                with open(file_path, "wb") as saliency:
                    saliency.write(zip_ref.read(file))

            elif "pascalFix.mat" in file:
                loaded_zip = io.BytesIO(zip_ref.read(file))

                with h5py.File(loaded_zip, "r") as f:
                    fixations = np.array(f.get("fixCell"))[0]

                    fixations_list = []

                    for reference in fixations:
                        obj = np.array(f[reference])
                        obj = np.stack((obj[0], obj[1]), axis=-1)
                        fixations_list.append(obj)

            elif "pascalSize.mat" in file:
                loaded_zip = io.BytesIO(zip_ref.read(file))

                with h5py.File(loaded_zip, "r") as f:
                    sizes = np.array(f.get("sizeData"))
                    sizes = np.transpose(sizes, (1, 0))

    for idx, value in enumerate(fixations_list):
        size = [int(x) for x in sizes[idx]]
        fixations_map = np.zeros(size)

        for fixation in value:
            fixations_map[int(fixation[0]) - 1,
                          int(fixation[1]) - 1] = 1

        file_name = str(idx + 1) + ".png"
        file_path = fixations_path + file_name

        imsave(file_path, fixations_map, cmap="gray")

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_osie(data_path):
    """Downloads the OSIE dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.
    """

    print(">> Downloading OSIE dataset...", end="", flush=True)

    default_path = data_path + "osie/"
    stimuli_path = default_path + "stimuli/"
    fixations_path = default_path + "fixations/"
    saliency_path = default_path + "saliency/"

    os.makedirs(stimuli_path, exist_ok=True)
    os.makedirs(fixations_path, exist_ok=True)
    os.makedirs(saliency_path, exist_ok=True)

    url = "https://github.com/NUS-VIP/predicting-human-gaze-beyond-pixels/archive/master.zip"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            file_name = os.path.basename(file)

            if file.endswith(".jpg") and "data/stimuli" in file:
                file_path = stimuli_path + file_name

                with open(file_path, "wb") as stimulus:
                    stimulus.write(zip_ref.read(file))

            elif file_name == "fixations.mat":
                loaded_zip = io.BytesIO(zip_ref.read(file))

                loaded_mat = loadmat(loaded_zip)["fixations"]

                for idx, value in enumerate(loaded_mat):
                    subjects = value[0][0][0][1]

                    fixations_map = np.zeros((600, 800))

                    for subject in subjects:
                        x_vals = subject[0][0][0][0][0]
                        y_vals = subject[0][0][0][1][0]

                        fixations = np.stack((y_vals, x_vals), axis=-1)
                        fixations = fixations.astype(int)

                        fixations_map[fixations[:, 0],
                                      fixations[:, 1]] = 1

                    file_name = str(1001 + idx) + ".png"

                    saliency_map = gaussian_filter(fixations_map, 16)

                    imsave(saliency_path + file_name, saliency_map, cmap="gray")
                    imsave(fixations_path + file_name, fixations_map, cmap="gray")

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_fiwi(data_path):
    """Downloads the FIWI dataset. Three folders are then created that
       contain the stimuli, binary fixation maps, and blurred saliency
       distributions respectively.

    Args:
        data_path (str): Defines the path where the dataset will be
                         downloaded and extracted to.
    """

    print(">> Downloading FIWI dataset...", end="", flush=True)

    default_path = data_path + "fiwi/"
    stimuli_path = default_path + "stimuli/"
    fixations_path = default_path + "fixations/"
    saliency_path = default_path + "saliency/"

    os.makedirs(stimuli_path, exist_ok=True)
    os.makedirs(fixations_path, exist_ok=True)
    os.makedirs(saliency_path, exist_ok=True)

    url = "https://www.dropbox.com/s/30nxg2uwd1wpb80/webpage_dataset.zip?dl=1"
    urlretrieve(url, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            file_name = os.path.basename(file)

            if file.endswith(".png") and "stimuli" in file:
                file_path = stimuli_path + file_name

                with open(file_path, "wb") as stimulus:
                    stimulus.write(zip_ref.read(file))

            elif file.endswith(".png") and "all5" in file:
                loaded_zip = io.BytesIO(zip_ref.read(file))

                fixations = imread(loaded_zip)
                saliency = gaussian_filter(fixations, 30)

                imsave(saliency_path + file_name, saliency, cmap="gray")
                imsave(fixations_path + file_name, fixations, cmap="gray")

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def download_pretrained_weights(data_path, key):
    """Downloads the pre-trained weights for the VGG16 model when
       training or the MSI-Net when testing on new data instances.

    Args:
        data_path (str): Defines the path where the weights will be
                         downloaded and extracted to.
        key (str): Describes the type of model for which the weights will
                   be downloaded. This contains the device and dataset.

    .. seealso:: The code for downloading files from google drive is based
                 on the solution provided at [https://bit.ly/2JSVgMQ].
    """

    print(">> Downloading pre-trained weights...", end="", flush=True)

    os.makedirs(data_path, exist_ok=True)

    ids = {
        "vgg16_hybrid": "1ff0va472Xs1bvidCwRlW3Ctf7Hbyyn7p",
        "model_salicon_cpu": "1Xy9C72pcA8DO4CY0rc6B7wsuE9L9DDZY",
        "model_salicon_gpu": "1Th7fqVYx25ePMZz4LYsjNQWgAu8tJqwL",
        "model_mit1003_cpu": "1jsESjYtsTvkMqKftA4rdstfB7mSYw5Ec",
        "model_mit1003_gpu": "1P_tWxBl3igZlzcHGp5H3T3kzsOskWeG6",
        "model_cat2000_cpu": "1XxaEx7xxD6rHasQTa-VY7T7eVpGhMxuV",
        "model_cat2000_gpu": "1T6ChEGB6Mf02gKXrENjdeD6XXJkE_jHh",
        "model_dutomron_cpu": "14tuRZpKi8LMDKRHNVUylu6RuAaXLjHTa",
        "model_dutomron_gpu": "15LG_M45fpYC1pTwnwmArNTZw_Z3BOIA-",
        "model_pascals_cpu": "1af9IvBqFamKWx64Ror6ALivuKNioOVIf",
        "model_pascals_gpu": "1C-T-RQzX2SaiY9Nw1HmaSx6syyCt01Z0",
        "model_osie_cpu": "1JD1tvAqZGxj_gEGmIfoxb9dTe5HOaHj1",
        "model_osie_gpu": "1g8UPr1hGpUdOSWerRb751pZqiWBOZOCh",
        "model_fiwi_cpu": "19qj9nAjd5gVHLB71oRn_YfYDw5n4Uf2X",
        "model_fiwi_gpu": "12OpIMIi2IyDVaxkE2d37XO9uUsSYf1Ec"
    }

    url = "https://drive.google.com/uc?id=" + ids[key] + "&export=download"

    session = requests.Session()

    response = session.get(url, params={"id": id}, stream=True)
    token = _get_confirm_token(response)

    if token:
        params = {"id": id, "confirm": token}
        response = session.get(url, params=params, stream=True)

    _save_response_content(response, data_path + "tmp.zip")

    with zipfile.ZipFile(data_path + "tmp.zip", "r") as zip_ref:
        for file in zip_ref.namelist():
            zip_ref.extract(file, data_path)

    os.remove(data_path + "tmp.zip")

    print("done!", flush=True)

def _get_confirm_token(response):
    for key, value in response.cookies.items():
        if key.startswith("download_warning"):
            return value

    return None

def _save_response_content(response, file_path):
    chunk_size = 32768

    with open(file_path, "wb") as data:
        for chunk in response.iter_content(chunk_size):
            if chunk:
                data.write(chunk)

max_stars_repo_name: bondruy/sam-tensorflow

max_issues_repo_name: balabalayy/sam-tensorflow

max_forks_repo_name: balabalayy/sam-tensorflow

reason: import numpy,from scipy


text:
import re
import sys
import os
import cv2
import numpy as np
import escapi

def list_cameras():
    print("Available cameras:")

class VideoReader():
    def __init__(self, capture):
        self.cap = cv2.VideoCapture(capture)
        if self.cap is None:
            print("The video source cannot be opened")
            sys.exit(0)
    def is_open(self):
        return self.cap.isOpened()
    def is_ready(self):
        return True
    def read(self):
        return self.cap.read()
    def close(self):
        self.cap.release()

class CameraReader(VideoReader):
    def __init__(self, capture, width, height, fps):
        self.device = None
        self.width = width
        self.height = height
        self.fps = fps
        if os.name == 'nt':
            escapi.init()
            self.device = capture
            self.buffer = escapi.init_camera(self.device, self.width, self.height, self.fps)
            escapi.do_capture(self.device)
        else:
            super(CameraReader, self).__init__(capture)
            self.cap.set(3, width)
            self.cap.set(4, height)
    def is_open(self):
        if self.device is None:
            return super(CameraReader, self).is_open()
        return True
    def is_ready(self):
        if self.device is None:
            return super(CameraReader, self).is_ready()
        return escapi.is_capture_done(self.device)
    def read(self):
        if self.device is None:
            return super(CameraReader, self).read()
        if escapi.is_capture_done(self.device):
            image = escapi.read(self.device, self.width, self.height, self.buffer)
            escapi.do_capture(self.device)
            return True, image
        else:
            return False, None
    def close(self):
        if self.device is None:
            super(CameraReader, self).close()
        else:
            escapi.deinit_camera(self.device)

class RawReader:
    def __init__(self, width, height):
        self.width = int(width)
        self.height = int(height)

        if self.width < 1 or self.height < 1:
            print("No acceptable size was given for reading raw RGB frames.")
            sys.exit(0)

        self.len = self.width * self.height * 3
        self.open = True
    def is_open(self):
        return self.open
    def is_ready(self):
        return True
    def read(self):
        frame = bytearray()
        read_bytes = 0
        while read_bytes < self.len:
            bytes = sys.stdin.buffer.read(self.len)
            read_bytes += len(bytes)
            frame.extend(bytes)
        return True, np.frombuffer(frame, dtype=np.uint8).reshape((self.height, self.width, 3))
    def close(self):
        self.open = False

class InputReader():
    def __init__(self, capture, raw_rgb, width, height, fps):
        self.reader = None
        try:
            if raw_rgb > 0:
                self.reader = RawReader(width, height)
            elif os.path.exists(capture):
                self.reader = VideoReader(capture)
            elif capture == str(int(capture)):
                self.reader = CameraReader(int(capture), width, height, fps)
        except Exception as e:
            print("Error: " + str(e))

        if self.reader is None or not self.reader.is_open():
            print("There was no valid input.")
            sys.exit(0)
    def is_open(self):
        return self.reader.is_open()
    def is_ready(self):
        return self.reader.is_ready()
    def read(self):
        return self.reader.read()
    def close(self):
        self.reader.close()

max_stars_repo_name: Avatarchik/OpenSeeFace

max_issues_repo_name: Avatarchik/OpenSeeFace

max_forks_repo_name: Avatarchik/OpenSeeFace


text:
import numpy as np
from qm import SimulationConfig, LoopbackInterface
from qm.qua import *

class TimeDiffCalibrator:
    @staticmethod
    def _default_config(freq, con_name):
        return {
            "version": 1,
            "controllers": {
                con_name: {
                    "type": "opx1",
                    "analog_outputs": {1: {"offset": 0.0}, 2: {"offset": 0.0}},
                    "digital_outputs": {},
                    "analog_inputs": {1: {"offset": 0.0}, 2: {"offset": 0.0}},
                }
            },
            "elements": {
                "rr": {
                    "mixInputs": {
                        "I": (con_name, 1),
                        "Q": (con_name, 2),
                        "lo_frequency": 0,
                        "mixer": "mixer",
                    },
                    "intermediate_frequency": freq,
                    "operations": {
                        "readout": "readout",
                    },
                    "outputs": {"out1": (con_name, 1), "out2": (con_name, 2)},
                    "time_of_flight": 32,
                    "smearing": 0,
                },
            },
            "pulses": {
                "readout": {
                    "operation": "measurement",
                    "length": 1000,
                    "waveforms": {
                        "I": "const_wf",
                        "Q": "zero_wf",
                    },
                    "integration_weights": {
                        "integW1": "integW1",
                        "integW2": "integW2",
                    },
                    "digital_marker": "ON",
                },
            },
            "waveforms": {
                "const_wf": {"type": "constant", "sample": 0.4},
                "zero_wf": {"type": "constant", "sample": 0.0},
            },
            "digital_waveforms": {"ON": {"samples": [(1, 0)]}},
            "integration_weights": {
                "integW1": {
                    "cosine": [1.0] * int(1000 / 4),
                    "sine": [0.0] * int(1000 / 4),
                },
                "integW2": {
                    "cosine": [0.0] * int(1000 / 4),
                    "sine": [1.0] * int(1000 / 4),
                },
            },
            "mixers": {
                "mixer": [
                    {
                        "intermediate_frequency": freq,
                        "lo_frequency": 0,
                        "correction": [1, 0, 0, 1],
                    }
                ]
            },
        }

    @staticmethod
    def calibrate(qmm, con_name):
        with program() as cal_phase:
            I1 = declare(fixed)
            Q1 = declare(fixed)
            I2 = declare(fixed)
            Q2 = declare(fixed)
            I = declare(fixed)
            Q = declare(fixed)
            n = declare(int)
            adc = declare_stream(adc_trace=True)
            with for_(n, 1, n < 1e4, n + 1):
                reset_phase("rr")
                measure(
                    "readout",
                    "rr",
                    "adc",
                    demod.full("integW1", I1, "out1"),
                    demod.full("integW2", Q1, "out1"),
                    demod.full("integW1", I2, "out2"),
                    demod.full("integW2", Q2, "out2"),
                )
                assign(I, I1 + Q2)
                assign(Q, -Q1 + I2)
                save(I, "I")
                save(Q, "Q")
            with stream_processing():
                adc.input1().save_all("adc_input1")
                adc.input2().save_all("adc_input2")
        freq = 8.4567e3
        qm = qmm.open_qm(TimeDiffCalibrator._default_config(freq, con_name))
        job = qm.simulate(
            cal_phase,
            SimulationConfig(
                500,
                simulation_interface=LoopbackInterface(
                    [("con1", 1, "con1", 1), ("con2", 2, "con2", 2)]
                ),
            ),
        )

        adc1 = np.mean(job.result_handles.adc_input1.fetch_all()["value"], axis=0)
        adc2 = np.mean(job.result_handles.adc_input2.fetch_all()["value"], axis=0)
        # adc_ts = job.result_handles.adc_input1.fetch_all()['timestamp']
        adc_ts = np.arange(0, len(adc1))
        I = np.mean(job.result_handles.get("I").fetch_all()["value"])
        Q = np.mean(job.result_handles.get("Q").fetch_all()["value"])

        sig = (adc1 + 1j * adc2) * np.exp(-1j * 2 * np.pi * freq * 1e-9 * adc_ts)
        d = np.sum(sig)

        I_ = np.real(d)
        Q_ = np.imag(d)

        time_diff_ns = np.angle((I + 1j * Q) / (I_ + 1j * Q_)) / 1e-9 / 2 / np.pi / freq
        time_diff = np.round(time_diff_ns / 4) * 4
        return time_diff

max_stars_repo_name: qua-platform/qua-libs

max_issues_repo_name: qua-platform/qua-libs

max_forks_repo_name: qua-platform/qua-libs


text:
# gym and np
from gym.spaces import Discrete
import numpy as np

# ray
from ray.rllib.models import ModelCatalog
from ray.rllib.models.torch.misc import SlimFC
from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
from ray.rllib.utils.framework import get_activation_fn, try_import_torch
torch, nn = try_import_torch()
from models.impala_cnn_torch import ResidualBlock, ConvSequence
from ray.rllib.utils.annotations import override
# aug
import kornia
# ae
from models import make_encoder, make_decoder

def weight_init(m):
    """Custom weight init for Conv2D and Linear layers."""
    if isinstance(m, nn.Linear):
        nn.init.orthogonal_(m.weight.data)
        m.bias.data.fill_(0.0)
    elif isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
        # delta-orthogonal init from https://arxiv.org/pdf/1806.05393.pdf
        assert m.weight.size(2) == m.weight.size(3)
        m.weight.data.fill_(0.0)
        m.bias.data.fill_(0.0)
        mid = m.weight.size(2) // 2
        gain = nn.init.calculate_gain('relu')
        nn.init.orthogonal_(m.weight.data[:, :, mid, mid], gain)

class SACAETorchModel(TorchModelV2, nn.Module):
    """Extension of standard TorchModelV2 for SAC.

    Data flow:
        obs -> forward() -> model_out
        model_out -> get_policy_output() -> pi(s)
        model_out, actions -> get_q_values() -> Q(s, a)
        model_out, actions -> get_twin_q_values() -> Q_twin(s, a)

    Note that this class by itself is not a valid model unless you
    implement forward() in a subclass."""

    def __init__(self,
                 obs_space,
                 action_space,
                 num_outputs,
                 model_config,
                 name,
                 actor_hidden_activation="relu",
                 actor_hiddens=(256, 256),
                 critic_hidden_activation="relu",
                 critic_hiddens=(256, 256),
                 twin_q=False,
                 initial_alpha=1.0,
                 target_entropy=None,
                 # customs
                 embed_dim = 50,
                 augmentation=False,
                 aug_num=2,
                 max_shift=4,
                 encoder_feature_dim=50,
                 num_layers=4,
                 num_filters=32,
                 decoder_type='pixel',
                 encoder_type='pixel',
                 **kwargs):
        """Initialize variables of this model.

        Extra model kwargs:
            actor_hidden_activation (str): activation for actor network
            actor_hiddens (list): hidden layers sizes for actor network
            critic_hidden_activation (str): activation for critic network
            critic_hiddens (list): hidden layers sizes for critic network
            twin_q (bool): build twin Q networks.
            initial_alpha (float): The initial value for the to-be-optimized
                alpha parameter (default: 1.0).
            target_entropy (Optional[float]): An optional fixed value for the
                SAC alpha loss term. None or "auto" for automatic calculation
                of this value according to [1] (cont. actions) or [2]
                (discrete actions).

        Note that the core layers for forward() are not defined here, this
        only defines the layers for the output heads. Those layers for
        forward() should be defined in subclasses of SACModel.
        """
        TorchModelV2.__init__(self, obs_space, action_space,
                                            num_outputs, model_config, name)
        nn.Module.__init__(self)

        self.action_dim = action_space.n
        self.discrete = True
        self.action_outs = q_outs = self.action_dim
        self.action_ins = None  # No action inputs for the discrete case.
        self.embed_dim = embed_dim

        h, w, c = obs_space.shape
        shape = (c, h, w)
        obs_shape = shape

        # obs embedding
        # conv_seqs = []
        # for out_channels in [16, 32, 32]:
        #     conv_seq = ConvSequence(shape, out_channels)
        #     shape = conv_seq.get_output_shape()
        #     conv_seqs.append(conv_seq)
        # self.conv_seqs = nn.ModuleList(conv_seqs)
        # self.hidden_fc = nn.Linear(in_features=shape[0] * shape[1] * shape[2], out_features=embed_dim)

        # Build the policy network
        # TODO: fix func input
        self.actor_encoder = make_encoder(
            encoder_type, obs_shape, encoder_feature_dim, num_layers, num_filters
        )

        self.critic_encoder = make_encoder(
            encoder_type, obs_shape, encoder_feature_dim, num_layers, num_filters
        )

        # tie encoders between actor and critic
        self.actor_encoder.copy_conv_weights_from(self.critic_encoder)

        self.action_model = nn.Sequential()
        # img -> embedding
        ins = embed_dim
        act = get_activation_fn(
            actor_hidden_activation, framework="torch")
        init = nn.init.xavier_uniform_
        outs = self.actor_encoder.feature_dim
        # embedding to autoencoder embed
        self.action_model.add_module(
                "action_{}".format('e'),
                SlimFC(ins, outs, initializer=init, activation_fn=act)
            )
        ins = outs
        # add trunk model
        for i, n in enumerate(actor_hiddens):
            self.action_model.add_module(
                "action_{}".format(i),
                SlimFC(ins, n, initializer=init, activation_fn=act)
            )
            ins = n
        self.action_model.add_module(
            "action_out",
            SlimFC(ins, self.action_outs, initializer=init, activation_fn=None)
        )

        # Build the Q-net(s), including target Q-net(s).
        def build_q_net(name_):

            act = get_activation_fn(
                critic_hidden_activation, framework="torch")
            init = nn.init.xavier_uniform_
            # For discrete actions, only obs.
            q_net = nn.Sequential()
            ins = embed_dim
            # embed to encoder embed
            outs = self.critic_encoder.feature_dim
            q_net.add_module(
                    "{}_hidden_{}".format(name_, "e"),
                    SlimFC(ins, outs, initializer=init, activation_fn=act)
                )
            ins = outs

            for i, n in enumerate(critic_hiddens):
                q_net.add_module(
                    "{}_hidden_{}".format(name_, i),
                    SlimFC(ins, n, initializer=init, activation_fn=act)
                )
                ins = n

            q_net.add_module(
                "{}_out".format(name_),
                SlimFC(ins, q_outs, initializer=init, activation_fn=None)
            )
            return q_net

        self.q_net = build_q_net("q")
        if twin_q:
            self.twin_q_net = build_q_net("twin_q")
        else:
            self.twin_q_net = None

        # temperature tensor
        self.log_alpha = torch.tensor(
            data=[np.log(initial_alpha)],
            dtype=torch.float32,
            requires_grad=True)

        # Auto-calculate the target entropy.
        if target_entropy is None or target_entropy == "auto":
            # See hyperparams in [2] (README.md).
            target_entropy = 0.98 * np.array(
                -np.log(1.0 / action_space.n), dtype=np.float32)

        self.target_entropy = torch.tensor(
            data=[target_entropy], dtype=torch.float32, requires_grad=False)

        # device = 0
        # make decoder
        self.decoder = None
        if decoder_type != 'identity':
            # create decoder
            self.decoder = make_decoder(
                decoder_type, obs_shape, encoder_feature_dim, num_layers,
                num_filters
            )
            self.decoder.apply(weight_init)

        # NOTE: custom fields
        self.augmentation = augmentation
        self.aug_num = aug_num
        # NOTE: augmentation
        if augmentation:
            obs_shape = obs_space.shape[-2]
            self.trans = nn.Sequential(
                nn.ReplicationPad2d(max_shift),
                kornia.augmentation.RandomCrop((obs_shape, obs_shape))
            )

    @override(TorchModelV2)
    def forward(self, input_dict, state, seq_lens):
        """ return embedding value
        """
        detach_encoder = False
        # x = self.get_embeddings(input_dict, state, seq_lens)
        x = self.actor_encoder(input_dict["obs"].permute(0,3,1,2), detach=detach_encoder)
        logits = self.get_policy_output(x)
        # only need value during training
        if input_dict["is_training"]:
            value = self.get_q_values(x)
            self._value = value.squeeze(1)
        return logits, state

    @override(TorchModelV2)
    def value_function(self):
        assert self._value is not None, "must call forward() first"
        return self._value

    def get_embeddings(self, input_dict, state, seq_lens, permute=True):
        """ encode observations
        """
        x = input_dict["obs"].float()
        x = x / 255.0  # scale to 0-1
        if permute:
            x = x.permute(0, 3, 1, 2)  # NHWC => NCHW
        for conv_seq in self.conv_seqs:
            x = conv_seq(x)
        x = torch.flatten(x, start_dim=1)
        x = nn.functional.relu(x)
        x = self.hidden_fc(x)
        x = nn.functional.relu(x)
        return x

    def get_q_values(self, model_out, actions=None):
        """Return the Q estimates for the most recent forward pass.

        This implements Q(s, a).

        Arguments:
            model_out (Tensor): obs embeddings from the model layers, of shape
                [BATCH_SIZE, num_outputs].
            actions (Optional[Tensor]): Actions to return the Q-values for.
                Shape: [BATCH_SIZE, action_dim]. If None (discrete action
                case), return Q-values for all actions.

        Returns:
            tensor of shape [BATCH_SIZE].
        """
        if actions is not None:
            return self.q_net(torch.cat([model_out, actions], -1))
        else:
            return self.q_net(model_out)

    def get_twin_q_values(self, model_out, actions=None):
        """Same as get_q_values but using the twin Q net.

        This implements the twin Q(s, a).

        Arguments:
            model_out (Tensor): obs embeddings from the model layers, of shape
                [BATCH_SIZE, num_outputs].
            actions (Optional[Tensor]): Actions to return the Q-values for.
                Shape: [BATCH_SIZE, action_dim]. If None (discrete action
                case), return Q-values for all actions.

        Returns:
            tensor of shape [BATCH_SIZE].
        """
        if actions is not None:
            return self.twin_q_net(torch.cat([model_out, actions], -1))
        else:
            return self.twin_q_net(model_out)

    def get_policy_output(self, model_out):
        """Return the action output for the most recent forward pass.

        This outputs the support for pi(s). For continuous action spaces, this
        is the action directly. For discrete, is is the mean / std dev.

        Arguments:
            model_out (Tensor): obs embeddings from the model layers, of shape
                [BATCH_SIZE, num_outputs].

        Returns:
            tensor of shape [BATCH_SIZE, action_out_size]
        """
        return self.action_model(model_out)

    def policy_variables(self):
        """Return the list of variables for the policy net."""

        return list(self.action_model.parameters())

    def q_variables(self):
        """Return the list of variables for Q / twin Q nets."""

        return list(self.q_net.parameters()) + \
            (list(self.twin_q_net.parameters()) if self.twin_q_net else [])

# Register model in ModelCatalog
ModelCatalog.register_custom_model("sac_ae", SACAETorchModel)

max_stars_repo_name: Justin-Yuan/efficient-rl

max_issues_repo_name: Justin-Yuan/efficient-rl

max_forks_repo_name: Justin-Yuan/efficient-rl