text:
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np

class encoderInitial_point(nn.Module):
    def __init__(self):
        super(encoderInitial_point, self).__init__()
        # Input should be segmentation, image with environment map, image with point light + environment map
        self.conv1 = nn.Conv2d(in_channels=4, out_channels=32, kernel_size=6, stride=2, padding=2, bias=False)
        self.bn1 = nn.BatchNorm2d(32)
        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(64)
        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn3 = nn.BatchNorm2d(128)
        self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn4 = nn.BatchNorm2d(256)
        self.conv5 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn5 = nn.BatchNorm2d(256)
        self.conv6 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn6 = nn.BatchNorm2d(512)

    def forward(self, x):
        x1 = F.relu(self.bn1(self.conv1(x)), True )
        x2 = F.relu(self.bn2(self.conv2(x1)), True )
        x3 = F.relu(self.bn3(self.conv3(x2)), True )
        x4 = F.relu(self.bn4(self.conv4(x3)), True )
        x5 = F.relu(self.bn5(self.conv5(x4)), True )
        x = F.relu(self.bn6(self.conv6(x5)), True )
        return x1, x2, x3, x4, x5, x

class encoderInitial(nn.Module):
    def __init__(self):
        super(encoderInitial, self).__init__()
        # Input should be segmentation, image with environment map, image with point light + environment map
        self.conv1 = nn.Conv2d(in_channels=7, out_channels=32, kernel_size=6, stride=2, padding=2, bias=False)
        self.bn1 = nn.BatchNorm2d(32)
        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(64)
        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn3 = nn.BatchNorm2d(128)
        self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn4 = nn.BatchNorm2d(256)
        self.conv5 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn5 = nn.BatchNorm2d(256)
        self.conv6 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn6 = nn.BatchNorm2d(512)

    def forward(self, x):
        x1 = F.relu(self.bn1(self.conv1(x)), True )
        x2 = F.relu(self.bn2(self.conv2(x1)), True )
        x3 = F.relu(self.bn3(self.conv3(x2)), True )
        x4 = F.relu(self.bn4(self.conv4(x3)), True )
        x5 = F.relu(self.bn5(self.conv5(x4)), True )
        x = F.relu(self.bn6(self.conv6(x5)), True )
        return x1, x2, x3, x4, x5, x

class decoderInitial(nn.Module):
    def __init__(self, mode=0):
        super(decoderInitial, self).__init__()
        # branch for normal prediction
        self.dconv0 = nn.ConvTranspose2d(in_channels=512, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn0 = nn.BatchNorm2d(256)
        self.dconv1 = nn.ConvTranspose2d(in_channels=256+256, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn1 = nn.BatchNorm2d(256)
        self.dconv2 = nn.ConvTranspose2d(in_channels=256+256, out_channels=128, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn2 = nn.BatchNorm2d(128)
        self.dconv3 = nn.ConvTranspose2d(in_channels=128+128, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn3 = nn.BatchNorm2d(64)
        self.dconv4 = nn.ConvTranspose2d(in_channels=64+64,  out_channels=32, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn4 = nn.BatchNorm2d(32)
        self.dconv5 = nn.ConvTranspose2d(in_channels=32+32, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn5 = nn.BatchNorm2d(64)

        self.convFinal = nn.Conv2d(in_channels=64, out_channels = 3, kernel_size = 5, stride=1, padding=2, bias=True)
        # Mode 0: Albedo
        # Mode 1: Normal
        # Mode 2: Roughness
        # Mode 3: Depth
        self.mode = mode
        assert( mode >= 0 and mode <= 3)

    def forward(self, x1, x2, x3, x4, x5, x):
        x_d1 = F.relu( self.dbn0(self.dconv0(x) ), True)
        x_d1_next = torch.cat( (x_d1, x5), dim = 1)
        x_d2 = F.relu( self.dbn1(self.dconv1(x_d1_next) ), True)
        x_d2_next = torch.cat( (x_d2, x4), dim = 1)
        x_d3 = F.relu( self.dbn2(self.dconv2(x_d2_next) ), True)
        x_d3_next = torch.cat( (x_d3, x3), dim = 1)
        x_d4 = F.relu( self.dbn3(self.dconv3(x_d3_next) ), True)
        x_d4_next = torch.cat( (x_d4, x2), dim = 1)
        x_d5 = F.relu( self.dbn4(self.dconv4(x_d4_next) ), True)
        x_d5_next = torch.cat( (x_d5, x1), dim = 1)
        x_d6 = F.relu( self.dbn5(self.dconv5(x_d5_next) ), True)
        x_orig  = torch.tanh( self.convFinal(x_d6) )
        if self.mode == 0:
            x_out = x_orig
        elif self.mode == 1:
            norm = torch.sqrt(torch.sum(x_orig * x_orig, dim=1).unsqueeze(1) ).expand_as(x_orig);
            x_out = x_orig / norm
        elif self.mode == 2:
            x_out = torch.mean(x_orig, dim=1).unsqueeze(1)
        elif self.mode == 3:
            x_out = torch.mean(x_orig, dim=1).unsqueeze(1)
            x_out = 1 / (0.4 * (x_out + 1) + 0.25 )
        return x_out

class envmapInitial(nn.Module):
    def __init__(self, numCoef = 9):
        super(envmapInitial, self).__init__()
        self.conv =nn.Conv2d(in_channels = 512, out_channels=1024, kernel_size=4, padding=0, stride=1)
        self.bn = nn.BatchNorm2d(1024)
        self.numCoef = numCoef

        self.regression = nn.Sequential(
                    nn.Linear(1024, 1024),
                    nn.ReLU(True),
                    nn.Dropout(0.25),
                    nn.Linear(1024, self.numCoef * 3)
                )

    def forward(self, x):
        x = F.relu(self.bn(self.conv(x) ) )
        x = x.view(x.size(0), x.size(1), -1)
        x = torch.mean(x, dim=2)
        x = x.view(x.size(0), -1)
        x = torch.tanh(self.regression(x))
        x = x.view( x.size(0), 3, self.numCoef)
        return x

class globalIllumination(nn.Module):
    def __init__(self):
        super(globalIllumination, self).__init__()
        # Encoder, the input will be albedo: 3, normal: 3, depth: 1, roughness: 1, n bounce: 3 segmentation: 1

        self.conv1 = nn.Conv2d(in_channels=12, out_channels=32, kernel_size=6, stride=2, padding=2, bias=False)
        #self.conv1 = nn.Conv2d(in_channels=12, out_channels=32, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(32)
        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=6, dilation=3, stride=2, padding=7, bias=False)
        #self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(64)
        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=6, dilation=3, stride=2, padding=7, bias=False)
        #self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn3 = nn.BatchNorm2d(128)
        self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn4 = nn.BatchNorm2d(256)
        self.conv5 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn5 = nn.BatchNorm2d(256)
        # Decoder
        self.dconv1 = nn.ConvTranspose2d(in_channels=256, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn1 = nn.BatchNorm2d(256)
        self.dconv2 = nn.ConvTranspose2d(in_channels=256 + 256, out_channels=128, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn2 = nn.BatchNorm2d(128)
        self.dconv3 = nn.ConvTranspose2d(in_channels=128 + 128, out_channels=64, kernel_size=6, dilation=3, stride=2, padding=7, bias=False)
        #self.dconv3 = nn.ConvTranspose2d(in_channels=128 + 128, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn3 = nn.BatchNorm2d(64)
        self.dconv4 = nn.ConvTranspose2d(in_channels=64 + 64,  out_channels=32, kernel_size=6, dilation=3, stride=2, padding=7, bias=False)
        #self.dconv4 = nn.ConvTranspose2d(in_channels=64 + 64, out_channels=32, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn4 = nn.BatchNorm2d(32)
        self.dconv5 = nn.ConvTranspose2d(in_channels=32 + 32, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn5 = nn.BatchNorm2d(64)
        self.convFinal = nn.Conv2d(in_channels=64, out_channels = 3, kernel_size = 5, stride=1, padding=2, bias=True)
        #self.convFinal = nn.Conv2d(in_channels=32, out_channels = 3, kernel_size = 3, stride=1, padding=1, bias=True)

    def forward(self, x):
        x1 = F.relu(self.bn1(self.conv1(x)), True )
        x2 = F.relu(self.bn2(self.conv2(x1)), True )
        x3 = F.relu(self.bn3(self.conv3(x2)), True )
        x4 = F.relu(self.bn4(self.conv4(x3)), True )
        x5 = F.relu(self.bn5(self.conv5(x4)), True )
        x_d1 = F.relu( self.dbn1(self.dconv1(x5) ), True)
        x_d1_next = torch.cat( (x_d1, x4), dim = 1)
        x_d2 = F.relu( self.dbn2(self.dconv2(x_d1_next) ), True)
        x_d2_next = torch.cat( (x_d2, x3), dim = 1)
        x_d3 = F.relu( self.dbn3(self.dconv3(x_d2_next) ), True)
        x_d3_next = torch.cat( (x_d3, x2), dim = 1)
        x_d4 = F.relu( self.dbn4(self.dconv4(x_d3_next) ), True)
        x_d4_next = torch.cat( (x_d4, x1), dim = 1)
        x_d5 = F.relu( self.dbn5(self.dconv5(x_d4_next) ), True)
        x_out  = torch.tanh( self.convFinal(x_d5) )
        return x_out

class residualBlock(nn.Module):
    def __init__(self, nchannels=128):
        super(residualBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels=nchannels, out_channels=nchannels, kernel_size=3, dilation = 2, stride=1, padding=2, bias=False)
        self.bn1 = nn.BatchNorm2d(nchannels)
        self.conv2 = nn.Conv2d(in_channels=nchannels, out_channels=nchannels, kernel_size=3, dilation = 2, stride=1, padding=2, bias=False)
        self.bn2 = nn.BatchNorm2d(nchannels)

    def forward(self, x):
        y = F.relu(self.bn1(self.conv1(x) ), True)
        y = F.relu(self.bn2(self.conv2(y) ), True)
        return y+x

class refineEncoder(nn.Module):
    def __init__(self):
        super(refineEncoder, self).__init__()
        # Encoder: segmentation: 1
        # albedo: 3, normal: 3 roughness: 1, depth: 1, two input image 7, total chnum = 15
        self.conv1 = nn.Conv2d(in_channels=15, out_channels=64, kernel_size=6, stride=2, padding=2, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=6, stride=2, padding=2, bias=False)
        self.bn2 = nn.BatchNorm2d(128)
        self.res1 = residualBlock(128)
        self.res2 = residualBlock(128)
        self.res3 = residualBlock(128)

    def forward(self, x):
        x1 = F.relu(self.bn1(self.conv1(x) ) )
        x2 = F.relu(self.bn2(self.conv2(x1) ) )
        x3 = self.res1(x2)
        x3 = self.res2(x3)
        x3 = self.res3(x3)
        return x1, x3

class refineEncoder(nn.Module):
    def __init__(self):
        super(refineEncoder, self).__init__()
        # Encoder: the input will be error: 3 segmentation: 1
        # albedo: 3, normal: 3 roughness: 1, depth: 1, two input image 6, total chnum = 18
        self.conv1 = nn.Conv2d(in_channels=18, out_channels=64, kernel_size=6, stride=2, padding=2, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=6, stride=2, padding=2, bias=False)
        self.bn2 = nn.BatchNorm2d(128)
        self.res1 = residualBlock(128)
        self.res2 = residualBlock(128)
        self.res3 = residualBlock(128)

    def forward(self, x):
        x1 = F.relu(self.bn1(self.conv1(x) ) )
        x2 = F.relu(self.bn2(self.conv2(x1) ) )
        x3 = self.res1(x2)
        x3 = self.res2(x3)
        x3 = self.res3(x3)
        return x1, x3

class refineDecoder(nn.Module):
    def __init__(self, mode = 0):
        super(refineDecoder, self).__init__()
        self.res1 = residualBlock(128)
        self.res2 = residualBlock(128)
        self.res3 = residualBlock(128)
        self.dconv0 = nn.ConvTranspose2d(in_channels=128, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn0 = nn.BatchNorm2d(64)
        self.dconv1 = nn.ConvTranspose2d(in_channels=64 + 64, out_channels=64, kernel_size=4, stride=2, padding=1, bias=False)
        self.dbn1 = nn.BatchNorm2d(64)
        self.convFinal = nn.Conv2d(in_channels=64, out_channels = 3, kernel_size = 5, stride=1, padding=2, bias=True)
        # Mode 0: Albedo
        # Mode 1: Normal
        # Mode 2: Roughness
        # Mode 3: Depth
        self.mode = mode

    def forward(self, x1, x3):
        x3 = self.res1(x3)
        x3 = self.res2(x3)
        x3 = self.res3(x3)
        x_d0 = F.relu(self.dbn0(self.dconv0(x3) ), True)
        x_d0_next = torch.cat( (x1, x_d0), dim=1)
        x_d1 = F.relu(self.dbn1(self.dconv1(x_d0_next) ), True)
        x_orig = torch.tanh(self.convFinal(x_d1) )
        if self.mode == 0:
            x_out = x_orig
        elif self.mode == 1:
            norm = torch.sqrt(torch.sum(x_orig * x_orig, dim=1).unsqueeze(1) ).expand_as(x_orig);
            x_out = x_orig / norm
        elif self.mode == 2:
            x_out = torch.mean(x_orig, dim=1).unsqueeze(1)
        elif self.mode == 3:
            x_out = torch.mean(x_orig, dim=1).unsqueeze(1)
            x_out = 1 / (0.4 * (x_out + 1) + 0.25 )
        return x_out

class refineEnvDecoder(nn.Module):
    def __init__(self):
        super(refineEnvDecoder, self).__init__()
        self.numCoef = 9
        self.conv1 = nn.Conv2d(in_channels = 128, out_channels=256, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(256)
        self.conv2 = nn.Conv2d(in_channels = 256, out_channels=256, kernel_size = 4, stride=2, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(256)
        self.conv3 = nn.Conv2d(in_channels = 256, out_channels=512, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn3 = nn.BatchNorm2d(512)
        self.conv4 = nn.Conv2d(in_channels = 512, out_channels=512, kernel_size=4, stride=2, padding=1, bias=False)
        self.bn4 = nn.BatchNorm2d(512)
        self.pool = nn.AvgPool2d(kernel_size = 4)
        self.projection = nn.Sequential(
                    nn.Linear(3 * self.numCoef, 512),
                    nn.ReLU(True),
                    nn.Dropout(0.25),
                    nn.Linear(512, 512)
                )
        self.regression = nn.Sequential(
                    nn.Linear(1024, 512),
                    nn.ReLU(True),
                    nn.Dropout(0.25),
                    nn.Linear(512, self.numCoef * 3)
                )

    def forward(self, x, pred):
        x = F.relu(self.bn1(self.conv1(x) ) )
        x = F.relu(self.bn2(self.conv2(x) ) )
        x = F.relu(self.bn3(self.conv3(x) ) )
        x = F.relu(self.bn4(self.conv4(x) ) )
        x = self.pool(x)
        x = x.view(x.size(0), -1)
        pred = pred.view( pred.size(0), 3*self.numCoef)
        pred = self.projection(pred)
        x = torch.cat([pred, x], dim=1)
        x = torch.tanh(self.regression(x) )
        x = x.view( x.size(0), 3, self.numCoef)
        return x

class renderingLayer():
    def __init__(self, imSize=256, fov=60, F0=0.05, cameraPos = [0, 0, 0], lightPos = [0, 0, 0],
            lightPower=5.95, gpuId = 0, isCuda = True):
        self.imSize = imSize
        self.fov = fov/180.0 * np.pi
        self.F0 = F0
        self.cameraPos = np.array(cameraPos, dtype=np.float32).reshape([1, 3, 1, 1])
        self.lightPos = np.array(lightPos, dtype=np.float32).reshape([1, 3, 1, 1])
        self.lightPower = lightPower
        self.yRange = self.xRange = 1 * np.tan(self.fov/2)
        self.isCuda = isCuda
        x, y = np.meshgrid(np.linspace(-self.xRange, self.xRange, imSize),
                np.linspace(-self.yRange, self.yRange, imSize) )
        y = np.flip(y, axis=0)
        z = -np.ones((imSize, imSize), dtype=np.float32)

        pCoord = np.stack([x, y, z]).astype(np.float32)
        self.pCoord = pCoord[np.newaxis, :, :, :]
        v = self.cameraPos - self.pCoord
        l = self.lightPos  - self.pCoord
        v = v / np.sqrt(np.maximum(np.sum(v*v, axis=1), 1e-12)[:, np.newaxis, :, :] )
        l = l / np.sqrt(np.maximum(np.sum(l*l, axis=1), 1e-12)[:, np.newaxis, :, :] )

        v = v.astype(dtype = np.float32)
        l = l.astype(dtype = np.float32)
        h = (v + l) / 2;
        h = h / np.sqrt(np.maximum(np.sum(h*h, axis=1), 1e-12)[:, np.newaxis, :, :] )
        h = h.astype(dtype = np.float32)

        self.v = Variable(torch.from_numpy(v) )
        self.l = Variable(torch.from_numpy(l) )
        self.h = Variable(torch.from_numpy(h) )
        temp = Variable(torch.FloatTensor(1, 1, 1, 1) )
        self.pCoord = Variable(torch.from_numpy(self.pCoord) )
        self.lightPos = Variable(torch.from_numpy(self.lightPos) )

        if isCuda:
            self.v = self.v.cuda(gpuId)
            self.l = self.l.cuda(gpuId)
            self.h = self.h.cuda(gpuId)
            self.pCoord = self.pCoord.cuda(gpuId)
            temp = temp.cuda(gpuId)
            self.lightPos = self.lightPos.cuda(gpuId)

        vdh = torch.sum( (self.v * self.h), dim = 1)
        vdh = vdh.unsqueeze(0)
        temp.data[0] = 2.0
        self.frac0 = F0 + (1-F0) * torch.pow(temp.expand_as(vdh), (-5.55472*vdh-6.98316)*vdh)
        self.vdh = vdh
        self.gpuId = gpuId
        self.sDist = None

    def forward(self, diffusePred, normalPred, roughPred, distPred, segBatch):
        diffuseBatch = (diffusePred + 1.0)/2.0 / np.pi
        roughBatch = (roughPred + 1.0)/2.0

        k = (roughBatch + 1) * (roughBatch + 1) / 8.0
        alpha = roughBatch * roughBatch
        alpha2 = alpha * alpha

        ndv = torch.clamp(torch.sum(normalPred * self.v.expand_as(normalPred), dim = 1), 0, 1)
        ndh = torch.clamp(torch.sum(normalPred * self.h.expand_as(normalPred), dim = 1), 0, 1)
        ndl = torch.clamp(torch.sum(normalPred * self.l.expand_as(normalPred), dim = 1), 0, 1)

        if len(ndv.size()) == 3:
            ndv = ndv.unsqueeze(1)
            ndh = ndh.unsqueeze(1)
            ndl = ndl.unsqueeze(1)

        frac = alpha2 * self.frac0.expand_as(alpha)
        nom0 = ndh * ndh * (alpha2 - 1) + 1
        nom1 = ndv * (1 - k) + k
        nom2 = ndl * (1 - k) + k
        nom = torch.clamp(4*np.pi*nom0*nom0*nom1*nom2, 1e-6, 4*np.pi)
        specPred = frac / nom

        coord3D = self.pCoord.expand_as(diffuseBatch) * distPred.expand_as(diffuseBatch)
        dist2Pred = torch.sum( (self.lightPos.expand_as(coord3D) - coord3D) \
                * (self.lightPos.expand_as(coord3D) - coord3D), dim=1).unsqueeze(1)
        color = (diffuseBatch + specPred.expand_as(diffusePred) ) * ndl.expand_as(diffusePred) * \
                self.lightPower / torch.clamp(dist2Pred.expand_as(diffusePred), 1e-6)
        color = color * segBatch.expand_as(diffusePred)
        return torch.clamp(color, 0, 1)

    def forwardEnv(self, diffusePred, normalPred, roughPred, SHPred, segBatch):
        diffuseBatch = (diffusePred + 1) / 2.0 / np.pi
        batchSize = diffusePred.size(0)
        c1, c2, c3, c4, c5 = 0.429043, 0.511664, 0.743125, 0.886227, 0.247708
        L0, L1_1, L10, L11, L2_2, L2_1, L20, L21, L22 = torch.split(SHPred, 1, dim=2)
        nx, ny, nz = torch.split(normalPred, 1, dim=1)
        L0, L1_1, L10, L11 = L0.contiguous(), L1_1.contiguous(), L10.contiguous(), L11.contiguous()
        L2_2, L2_1, L20, L21, L22 = L2_2.contiguous(), L2_1.contiguous(), \
                L20.contiguous(), L21.contiguous(), L22.contiguous()
        L0 = L0.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L1_1 = -L1_1.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L10 = L10.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L11 = -L11.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L2_2 = L2_2.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L2_1 = -L2_1.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L20 = L20.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L21 = -L21.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        L22 = L22.view(batchSize, 3, 1, 1).expand([batchSize, 3, self.imSize, self.imSize] )
        nx = nx.expand([batchSize, 3, self.imSize, self.imSize] )
        ny = ny.expand([batchSize, 3, self.imSize, self.imSize] )
        nz = nz.expand([batchSize, 3, self.imSize, self.imSize] )

        x = c1*L22*nx + c1*L2_2*ny + c1*L21*nz + c2*L11
        y = c1*L2_2*nx - c1*L22*ny + c1*L2_1*nz + c2*L1_1
        z = c1*L21*nx + c1*L2_1*ny + c3*L20*nz + c2*L10
        w = c2*L11*nx + c2*L1_1*ny + c2*L10*nz + c4*L0 - c5*L20

        radiance = nx*x + ny*y + nz*z + w

        color = diffuseBatch * radiance * segBatch.expand_as(diffusePred)
        return torch.clamp(color, 0, 1)

max_stars_repo_name: YouweiLyu/SingleImageShapeAndSVBRDF

max_issues_repo_name: YouweiLyu/SingleImageShapeAndSVBRDF

max_forks_repo_name: YouweiLyu/SingleImageShapeAndSVBRDF


text:
# this script is to evaluate the success of the retrieval
# its a simple list of counts and percentages which should be printed out.
import numpy as np

def df_eval(df):
    # quick percentage func
    def pc(x, y):
        return(f'{np.round(x/y*100,1)}%')

    # count for any format retrival
    retrieved = 0
    # count for tagged and pdf retrieval
    criteria = 0

    for index, row in df.iterrows():
        if row['html'] == 1\
        or row['pdf'] ==1\
        or row['xml'] == 1\
        or row['plain'] == 1:
            #we retrieved at least one format
            retrieved +=1

        if (row['html'] == 1\
        or row['xml'] == 1)\
        and (row['pdf'] == 1):
        #we got at least one tagged version with the pdf format
            criteria +=1
    #sum of each format retreived
    html_c = sum(df['html'] == 1)
    pdf_c = sum(df['pdf'] == 1)
    xml_c = sum(df['xml'] == 1)
    plain_c = sum(df['plain'] == 1)

    #printing the result
    print(f'Out of {len(df)} articles\nwe got at least one version for {retrieved} = {pc(retrieved, len(df))}\n')
    print(f'We got a pdf and tagged version for {criteria} artilces = {pc(criteria,len(df))}\n')
    print(f'{html_c} HTMLs were retrieved = {pc(html_c, len(df))}')
    print(f'{pdf_c} PDFs were retrieved = {pc(pdf_c, len(df))}')
    print(f'{xml_c} XMLs were retrieved = {pc(xml_c, len(df))}')
    print(f'{plain_c} plain texts were retrieved = {pc(plain_c, len(df))}')

ext: py

max_stars_repo_name: biomedicalinformaticsgroup/cadmus

max_issues_repo_name: biomedicalinformaticsgroup/cadmus

max_forks_repo_name: biomedicalinformaticsgroup/cadmus

reason: import numpy


text:
'''2. Un programa pide la edad de 5 personas, se almacenan en 5 variables (no hemos visto arreglos). El programa entrega el promedio, la suma y la desviación estándar de las edades. Tanto el promedio como la desv. estándar deben ser calculados por su fórmula y no usando las funciones que trae python'''

import numpy

# Ingresamos la edad de 5 personas
edad1 = int(input('Ingrese edad persona 1: '))
edad2 = int(input('Ingrese edad persona 2: '))
edad3 = int(input('Ingrese edad persona 3: '))
edad4 = int(input('Ingrese edad persona 4: '))
edad5 = int(input('Ingrese edad persona 5: '))

# Suma
suma = edad1 + edad2 + edad3 + edad4 + edad5
print('Suma de edades:', suma)

# Promedio
# opcion 1
promedio1 = suma/5
print('Promedio1:', promedio1)

# opcion 2
promedio2 = (edad1 + edad2 + edad3 + edad4 + edad5)/5
print('Promedio2:', promedio2)

# desviación estándar
desv_numerador = (edad1 - promedio1)**2 + (edad2 - promedio1)**2 + \
    (edad3 - promedio1)**2 + (edad4 - promedio1)**2 + (edad5 - promedio1)**2

desv_estandar = (desv_numerador/5)**(1/2)
print('Desviación estándar: ', desv_estandar)

# No considerar aún ...
edades = [edad1, edad2, edad3, edad4, edad5]
desv = numpy.std(edades)
print(desv)

max_stars_repo_path: variables and data types/variables2.py

max_stars_repo_name: camipozas/python-exercises

max_issues_repo_path: variables and data types/variables2.py

max_issues_repo_name: camipozas/python-exercises

max_forks_repo_path: variables and data types/variables2.py

max_forks_repo_name: camipozas/python-exercises


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Apr  2 09:26:03 2015

@author: Diogo Silva

CUDA not implemented on this version.
"""

import numpy as np

import sys, traceback

class K_Means:

    def __init__(self,N=None,D=None,K=None):
        self.N = N
        self.D = D
        self.K = K

    def fit(self, data, K, iters=3, cuda=False):

        N,D = data.shape

        self.N = N
        self.D = D
        self.K = K

        self.centroids = self._init_centroids(data)

        if iters == 0:
            return

        for i in xrange(iters):
            dist_mat = self._calc_dists(data,self.centroids,cuda=cuda)
            assign,grouped_data = self._assign_data(data,dist_mat)
            self.centroids =  self._np_recompute_centroids(grouped_data)

    def _init_centroids(self,data):

        centroids = np.empty((self.K,self.D),dtype=data.dtype)
        random_init = np.random.randint(0,self.N,self.K)
        self.init_seed = random_init

        for k in xrange(self.K):
            centroids[k] = data[random_init[k]]

        self.centroids = centroids

        return centroids

    def _calc_dists(self,data,centroids,cuda=False):
        if cuda:
            dist_mat = self._cu_calc_dists(data,centroids,gridDim=None,
                                           blockDim=None)#,memManage='manual')
        else:
            dist_mat = self._np_calc_dists(data,centroids)

        return dist_mat

    def _py_calc_dists(data,centroids):
        N,D = data.shape
        K,cD = centroids.shape

        for n in range(N):
            for k in range(K):
                dist=0
                for d in range(dim):
                    diff = a[n,d]-b[k,d]
                    dist += diff ** 2
                c[n,k]=dist

    def _np_calc_dists(self,data,centroids):
        """
        NumPy implementation - much faster than vanilla Python
        """
        N,D = data.shape
        K,cD = centroids.shape

        dist_mat = np.empty((N,K),dtype=data.dtype)

        for k in xrange(K):
            dist = data - centroids[k]
            dist = dist ** 2
            dist_mat[:,k] = dist.sum(axis=1)

        return dist_mat

    def _assign_data(self,data,dist_mat):

        N,K = dist_mat.shape

        assign = np.argmin(dist_mat,axis=1)

        grouped_data=[[] for i in xrange(K)]

        for n in xrange(N):
            # add datum i to its assigned cluster assign[i]
            grouped_data[assign[n]].append(data[n])

        for k in xrange(K):
            grouped_data[k] = np.array(grouped_data[k])

        return assign,grouped_data

    def _np_recompute_centroids(self,grouped_data):

        # change to get dimension from class or search a non-empty cluster
        #dim = grouped_data[0][0].shape[1]
        dim = self.D
        K = len(grouped_data)

        centroids = np.empty((K,dim),dtype=grouped_data[0].dtype)

        for k in xrange(K):
            centroids[k] = np.mean(grouped_data[k],axis=0)

        return centroids

max_stars_repo_name: Chiroptera/QCThesis

max_issues_repo_name: Chiroptera/QCThesis

max_forks_repo_name: Chiroptera/QCThesis


text:
import os

import numpy as np
import scipy.io as sio

import pandas as pd
from openseapy.dataset import SNADataset

class CoreLoader:
    """
    Core class for data loading.
    """
    def __init__(self, amplify=1e12, time_row=None, sample_rate=None, use_generic_trace_id=True):
        """
        Parameters
        ----------
        amplify: flout, optional
            Factor to amplify signal values. Default is 1e12.
        time_row: int or None, optional
            Row id of the time vector. If None, the time vector will be generated based on the sample rate.
            Default is None.
        sample_rate: float or None, optional
            Sample rate of the data. If None, a time vector cannot be created or proofed. Default is None.
        use_generic_trace_id: bool, optional
            Should the loader generate a generic trace id. Default is True.
        """
        self.amplify = amplify
        self.time_row = time_row
        self.sample_rate = sample_rate
        self.use_generic_trace_id = use_generic_trace_id

    def _load(self, source_file):
        pass

    def __call__(self, source_file):
        return self._load(source_file)

class HekaMatLoader(CoreLoader):
    """
    Loader for Heka mat files.
    """
    def __init__(self, *args, **kwargs):
        """
        Parameters
        ----------
        amplify: flout, optional
            Factor to amplify signal values. Default is 1e12.
        time_row: int or None, optional
            Row id of the time vector. If None, the time vector will be generated based on the sample rate.
            Default is None.
        sample_rate: float or None, optional
            Sample rate of the data. If None, a time vector cannot be created or proofed. Default is None.
        use_generic_trace_id: bool, optional
            Should the loader generate a generic trace id. Default is True.
        """
        super(HekaMatLoader, self).__init__(*args, **kwargs)

    def _load(self, source_file):
        folder, filename = os.path.split(source_file)
        primary_name = os.path.splitext(filename)[0]

        mat_contents = sio.loadmat(source_file)

        traces_list = list(filter(lambda entry: entry[0] != '_', mat_contents.keys()))

        for trace_id, trace_name in enumerate(traces_list):
            y = mat_contents[trace_name]
            if self.time_row is None:
                if self.sample_rate is None:
                    raise AttributeError('If no time row is defined, sample rate has to be set!')
                time = np.arange(0, len(y) / self.sample_rate, 1 / self.sample_rate)
            else:
                time = mat_contents[trace_name][:, self.time_row]
                fs_file = np.median(1 / np.diff(mat_contents[trace_name][:, self.time_row]))

                if self.sample_rate is not None:
                    assert 0.01 > np.abs(fs_file / self.sample_rate - 1), \
                        "Problem with sample rate: {0:} != {1:}".format(self.sample_rate, fs_file)

            for idr, row in enumerate(y.T):
                if idr == self.time_row:
                    continue

                if self.use_generic_trace_id:
                    name = primary_name + '/t{:03d}'.format(trace_id) + '/r{:03d}'.format(idr)
                else:
                    name = primary_name + '/' + trace_name + '/r{:03d}'.format(idr)

                yield pd.Series(row * self.amplify, index=time, name=name)

class CSVLoader(CoreLoader):
    """
    Loader class for CSV data.
    """
    def __init__(self, *args, **kwargs):
        """
        Parameters
        ----------
        amplify: flout, optional
            Factor to amplify signal values. Default is 1e12.
        time_row: int or None, optional
            Row id of the time vector. If None, the time vector will be generated based on the sample rate.
            Default is None.
        sample_rate: float or None, optional
            Sample rate of the data. If None, a time vector cannot be created or proofed. Default is None.
        """
        super(CSVLoader, self).__init__(*args, **kwargs)

        if self.use_generic_trace_id is not True:
            raise ValueError('CSVLoader do not support non generic trace ids!')

    def _load(self, source_file, **kwargs):
        folder, filename = os.path.split(source_file)
        primary_name = os.path.splitext(filename)[0]

        y = pd.read_csv(source_file, **kwargs).values

        if self.time_row is None:
            if self.sample_rate is None:
                raise AttributeError('If no time row is defined, sample rate has to be set!')
            time = np.arange(0, len(y) / self.sample_rate, 1 / self.sample_rate)
        else:
            time = y[:, self.time_row]
            fs_file = np.median(1 / np.diff(y[:, self.time_row]))

            if self.sample_rate is not None:
                assert 0.01 > np.abs(fs_file / self.sample_rate - 1), \
                    "Problem with sample rate: {0:} != {1:}".format(self.sample_rate, fs_file)

        for idr, row in enumerate(y.T):
            if idr == self.time_row:
                continue

            name = primary_name + '/t{:03d}'.format(0) + '/r{:03d}'.format(idr)

            yield pd.Series(row * self.amplify, index=time, name=name)

def collect_data(loader, sources_dict, target_file, append=False):
    """
    Collect raw data.

    Parameters
    ----------
    loader: CoreLoader
        loader object
    sources_dict: dictionary
        dictionary with information about the files (key) and additional meta information (value)
    target_file: str
        path to the file to store the data
    append: bool, optional
        If true, the data will be appended, else the file will be overwritten. Default False.

    Returns
    -------
    SNADataset
    """

    config_columns = sources_dict[next(iter(sources_dict))].keys()
    file_registration = pd.DataFrame([], columns=['filename', 'folder', *list(config_columns)])
    trace_registration = pd.DataFrame([], columns=['primary_name'])

    for source_file in sources_dict:
        folder, filename = os.path.split(source_file)
        primary_name = os.path.splitext(filename)[0]

        config = sources_dict[source_file]

        file_registration.loc[primary_name] = [filename, folder, *list(config.values())]

        for series in loader(source_file):
            series_name = series.name
            trace_registration.loc[series_name] = [primary_name]

            series.to_hdf(target_file, key='series/' + series_name)

    if append:
        try:
            previous_file_registration = pd.read_hdf(target_file, key='data_registration')
            previous_trace_registration = pd.read_hdf(target_file, key='trace_registration')

            file_registration = previous_file_registration.append(file_registration)
            trace_registration = previous_trace_registration.append(trace_registration)
        except KeyError:
            pass

    file_registration.to_hdf(target_file, key='data_registration')
    trace_registration.to_hdf(target_file, key='trace_registration')

    return SNADataset(target_file)

max_stars_repo_name: Digusil/snaa

max_issues_repo_name: Digusil/snaa

max_forks_repo_name: Digusil/snaa

reason: import numpy,import scipy


text:
from .vectors import Vector3D
import numpy as np

class Particle():
    def __init__(self, x, y, z, weight, elasticity, speed, acceleration):
        self._pos = np.array([x,y,z])
        self.weight = weight
        self.elasticity = elasticity
        self.elasticity = elasticity
        self.acceleration = acceleration
        self.movement_vectors = []

    def add_movement_vector(self, movement_vector):
        if isinstance(movement_vector, list):
            movement_vector = Vector3D(*movement_vector)

        self.movement_vectors.append(movement_vector)

    def move(self, time_frame):
        for mov_vec in self.movement_vectors:
            self.pos += mov_vec.norm_direction * mov_vec.intencity * time_frame

    @property
    def pos(self):
        return self._pos

    @pos.setter
    def pos(self, pos):
        self._pos = pos

max_stars_repo_name: jadsonlucio/Opengl-CG-Project

max_issues_repo_name: jadsonlucio/Opengl-CG-Project

max_forks_repo_name: jadsonlucio/Opengl-CG-Project


text:
from __future__ import absolute_import, division, print_function

from collections import defaultdict

import numpy as np

from glue.core import Subset
from glue.external.echo import (delay_callback, CallbackProperty,
                                HasCallbackProperties, CallbackList)
from glue.core.state import saver, loader

__all__ = ['State', 'StateAttributeCacheHelper',
           'StateAttributeLimitsHelper', 'StateAttributeSingleValueHelper']

@saver(CallbackList)
def _save_callback_list(items, context):
    return {'values': [context.id(item) for item in items]}

@loader(CallbackList)
def _load_callback_list(rec, context):
    return [context.object(obj) for obj in rec['values']]

class State(HasCallbackProperties):
    """
    A class to represent the state of a UI element. Initially this doesn't add
    anything compared to HasCallbackProperties, but functionality will be added
    over time.
    """

    def __init__(self, **kwargs):
        super(State, self).__init__()
        self.update_from_dict(kwargs)

    def update_from_state(self, state):
        """
        Update this state using the values from another state.

        Parameters
        ----------
        state : `~glue.core.state_objects.State`
            The state to use the values from
        """
        self.update_from_dict(state.as_dict())

    def update_from_dict(self, properties):
        """
        Update this state using the values from a dictionary of attributes.

        Parameters
        ----------
        properties : dict
            The dictionary containing attribute/value pairs.
        """

        if len(properties) == 0:
            return

        # Group properties into priority so as to be able to update them in
        # chunks and not fire off callback events between every single one.
        groups = defaultdict(list)
        for name in properties:
            if self.is_callback_property(name):
                groups[self._update_priority(name)].append(name)

        for priority in sorted(groups, reverse=True):
            with delay_callback(self, *groups[priority]):
                for name in groups[priority]:
                    setattr(self, name, properties[name])

    def as_dict(self):
        """
        Return the current state as a dictionary of attribute/value pairs.
        """
        properties = {}
        for name in dir(self):
            if self.is_callback_property(name):
                properties[name] = getattr(self, name)
        return properties

    def __gluestate__(self, context):
        return {'values': dict((key, context.id(value)) for key, value in self.as_dict().items())}

    def _update_priority(self, name):
        return 0

    @classmethod
    def __setgluestate__(cls, rec, context):
        properties = dict((key, context.object(value)) for key, value in rec['values'].items())
        result = cls(**properties)
        return result

class StateAttributeCacheHelper(object):
    """
    Generic class to help with caching values on a per-attribute basis

    Parameters
    ----------
    state : :class:`glue.core.state_objects.State`
        The state object with the callback properties to cache
    attribute : str
        The attribute name - this will be populated once a dataset is assigned
        to the helper
    cache : dict, optional
        A dictionary that can be used to hold the cache. This option can be used
        if a common cache should be shared between different helpers.
    kwargs
        Additional keyword arguments are taken to be values that should be
        used/cached. The key should be the name to be understood by sub-classes
        of this base class, and the value should be the name of the attribute
        in the state.
    """

    def __init__(self, state, attribute, cache=None, **kwargs):

        self._state = state
        self._attribute = attribute
        self._values = dict((key, kwargs[key]) for key in self.values_names if key in kwargs)
        self._modifiers = dict((key, kwargs[key]) for key in self.modifiers_names if key in kwargs)

        self._attribute_lookup = {'attribute': self._attribute}
        self._attribute_lookup.update(self._values)
        self._attribute_lookup.update(self._modifiers)
        self._attribute_lookup_inv = {v: k for k, v in self._attribute_lookup.items()}

        self._state.add_callback(self._attribute, self._update_attribute)

        self._state.add_global_callback(self._update_values)

        # NOTE: don't use self._cache = cache or {} here since if the initial
        #       cache is empty it will evaluate as False!
        if cache is None:
            self._cache = {}
        else:
            self._cache = cache

    @property
    def data_values(self):
        # For subsets in 'data' mode, we want to compute the limits based on
        # the full dataset, not just the subset.
        if isinstance(self.data, Subset):
            return self.data.data[self.component_id]
        else:
            return self.data[self.component_id]

    @property
    def data_component(self):
        # For subsets in 'data' mode, we want to compute the limits based on
        # the full dataset, not just the subset.
        if isinstance(self.data, Subset):
            return self.data.data.get_component(self.component_id)
        else:
            return self.data.get_component(self.component_id)

    def invalidate_cache(self):
        self._cache.clear()

    @property
    def data(self):
        if self.attribute is None:
            return None
        else:
            return self.attribute.parent

    @property
    def component_id(self):
        if self.attribute is None:
            return None
        else:
            return self.attribute

    def set_cache(self, cache):
        self._cache = cache
        self._update_attribute()

    def _update_attribute(self, *args):
        if self.component_id in self._cache:
            # The component ID already exists in the cache, so just revert to
            # that version of the values/settings.
            self.set(cache=False, **self._cache[self.component_id])
        else:
            # We need to compute the values for the first time
            self.update_values(attribute=self.component_id, use_default_modifiers=True)

    def _update_values(self, **properties):

        if hasattr(self, '_in_set'):
            if self._in_set:
                return
        if self.attribute is None:
            return
        properties = dict((self._attribute_lookup_inv[key], value)
                          for key, value in properties.items() if key in self._attribute_lookup_inv and self._attribute_lookup_inv[key] != 'attribute')
        if len(properties) > 0:
            self.update_values(**properties)

    def _modifiers_as_dict(self):
        return dict((prop, getattr(self, prop)) for prop in self.modifiers_names if prop in self._modifiers)

    def _values_as_dict(self):
        return dict((prop, getattr(self, prop)) for prop in self.values_names if prop in self._values)

    def _update_cache(self):
        if self.component_id is not None:
            self._cache[self.component_id] = {}
            self._cache[self.component_id].update(self._modifiers_as_dict())
            self._cache[self.component_id].update(self._values_as_dict())

    def __getattr__(self, attribute):
        if attribute in self._attribute_lookup:
            return getattr(self._state, self._attribute_lookup[attribute])
        else:
            raise AttributeError(attribute)

    def __setattr__(self, attribute, value):
        if attribute.startswith('_') or attribute not in self._attribute_lookup:
            return object.__setattr__(self, attribute, value)
        else:
            return setattr(self._state, self._attribute_lookup[attribute], value)

    def set(self, cache=True, **kwargs):

        self._in_set = True

        extra_kwargs = set(kwargs.keys()) - set(self.values_names) - set(self.modifiers_names)

        if len(extra_kwargs) > 0:
            raise ValueError("Invalid properties: {0}".format(extra_kwargs))

        with delay_callback(self._state, *self._attribute_lookup.values()):
            for prop, value in kwargs.items():
                setattr(self, prop, value)

        if cache:
            self._update_cache()

        self._in_set = False

class StateAttributeLimitsHelper(StateAttributeCacheHelper):
    """
    This class is a helper for attribute-dependent min/max level values. It
    is equivalent to AttributeLimitsHelper but operates on State objects and
    is GUI-independent.

    Parameters
    ----------
    attribute : str
        The attribute name - this will be populated once a dataset is assigned
        to the helper.
    percentile_subset : int
        How many points to use at most for the percentile calculation (using all
        values is highly inefficient and not needed)
    lower, upper : str
        The fields for the lower/upper levels
    percentile : ``QComboBox`` instance, optional
        The scale mode combo - this will be populated by presets such as
        Min/Max, various percentile levels, and Custom.
    log : bool
        Whether the limits are in log mode (in which case only positive values
        are used when finding the limits)

    Notes
    -----

    Once the helper is instantiated, the data associated with the helper can be
    set/changed with:

    >>> helper = AttributeLimitsHelper(...)
    >>> helper.data = data

    The data can also be passed to the initializer as described in the list of
    parameters above.
    """

    values_names = ('lower', 'upper')
    modifiers_names = ('log', 'percentile')

    def __init__(self, state, attribute, percentile_subset=10000, cache=None, **kwargs):

        super(StateAttributeLimitsHelper, self).__init__(state, attribute, cache=cache, **kwargs)

        self.percentile_subset = percentile_subset

        if self.attribute is not None:

            if (self.lower is not None and self.upper is not None and getattr(self, 'percentile', None) is None):
                # If the lower and upper limits are already set, we need to make
                # sure we don't override them, so we set the percentile mode to
                # custom if it isn't already set.
                self.set(percentile='Custom')
            else:
                # Otherwise, we force the recalculation or the fetching from
                # cache of the limits based on the current attribute
                self._update_attribute()

    def update_values(self, use_default_modifiers=False, **properties):

        if not any(prop in properties for prop in ('attribute', 'percentile', 'log')):
            self.set(percentile='Custom')
            return

        if use_default_modifiers:
            percentile = 100
            log = False
        else:
            percentile = self.percentile or 100
            log = self.log or False

        if percentile == 'Custom' or not hasattr(self, 'data') or self.data is None:

            self.set(percentile=percentile, log=log)

        else:

            exclude = (100 - percentile) / 2.

            data_values = self.data_values

            if data_values.size > self.percentile_subset:
                data_values = np.random.choice(data_values.ravel(), self.percentile_subset)

            if log:
                data_values = data_values[data_values > 0]
                if len(data_values) == 0:
                    self.set(lower=0.1, upper=1, percentile=percentile, log=log)
                    return

            try:
                lower = np.nanpercentile(data_values, exclude)
                upper = np.nanpercentile(data_values, 100 - exclude)
            except AttributeError:  # Numpy < 1.9
                data_values = data_values[~np.isnan(data_values)]
                lower = np.percentile(data_values, exclude)
                upper = np.percentile(data_values, 100 - exclude)

            if self.data_component.categorical:
                lower = np.floor(lower - 0.5) + 0.5
                upper = np.ceil(upper + 0.5) - 0.5

            self.set(lower=lower, upper=upper, percentile=percentile, log=log)

    def flip_limits(self):
        self.set(lower=self.upper, upper=self.lower)

class StateAttributeSingleValueHelper(StateAttributeCacheHelper):

    values_names = ('value',)
    modifiers_names = ()

    def __init__(self, state, attribute, function, mode='values', **kwargs):
        self._function = function
        super(StateAttributeSingleValueHelper, self).__init__(state, attribute, **kwargs)
        if self.attribute is not None:
            self._update_attribute()
        if mode in ('values', 'component'):
            self.mode = mode
        else:
            raise ValueError('mode should be one of "values" or "component"')

    def update_values(self, use_default_modifiers=False, **properties):
        if not any(prop in properties for prop in ('attribute',)) or self.data is None:
            self.set()
        else:
            if self.mode == 'values':
                arg = self.data_values
            else:
                arg = self.data_component
            self.set(value=self._function(arg))

class StateAttributeHistogramHelper(StateAttributeCacheHelper):

    values_names = ('lower', 'upper', 'n_bin')
    modifiers_names = ()

    def __init__(self, *args, **kwargs):

        self._max_n_bin = kwargs.pop('max_n_bin', 30)
        self._default_n_bin = kwargs.pop('default_n_bin', 15)

        common_n_bin_att = kwargs.pop('common_n_bin', None)

        super(StateAttributeHistogramHelper, self).__init__(*args, **kwargs)

        if common_n_bin_att is not None:
            if getattr(self._state, common_n_bin_att):
                self._common_n_bin = self._default_n_bin
            else:
                self._common_n_bin = None
            self._state.add_callback(common_n_bin_att, self._update_common_n_bin)
        else:
            self._common_n_bin = None

    def _apply_common_n_bin(self):
        for att in self._cache:
            cmp = self.data.get_component(att)
            if not cmp.categorical:
                self._cache[att]['n_bin'] = self._common_n_bin

    def _update_common_n_bin(self, common_n_bin):
        if common_n_bin:
            if self.data_component.categorical:
                self._common_n_bin = self._default_n_bin
            else:
                self._common_n_bin = self.n_bin
            self._apply_common_n_bin()
        else:
            self._common_n_bin = None

    def update_values(self, use_default_modifiers=False, **properties):

        if not any(prop in properties for prop in ('attribute', 'n_bin')) or self.data is None:
            self.set()
            return

        comp = self.data_component

        if 'n_bin' in properties:
            self.set()
            if self._common_n_bin is not None and not comp.categorical:
                self._common_n_bin = properties['n_bin']
                self._apply_common_n_bin()

        if 'attribute' in properties:

            if comp.categorical:

                n_bin = max(1, min(comp.categories.size, self._max_n_bin))
                lower = -0.5
                upper = lower + comp.categories.size

            else:

                if self._common_n_bin is None:
                    n_bin = self._default_n_bin
                else:
                    n_bin = self._common_n_bin

                values = self.data_values
                lower = np.nanmin(values)
                upper = np.nanmax(values)

            self.set(lower=lower, upper=upper, n_bin=n_bin)

if __name__ == "__main__":

    from glue.core import Data

    class TestState(object):

        layer = CallbackProperty()
        comp = CallbackProperty(1)
        lower = CallbackProperty()
        higher = CallbackProperty()
        log = CallbackProperty()
        scale = CallbackProperty()

    state = TestState()
    state.layer = Data(x=np.arange(10),
                       y=np.arange(10) / 3.,
                       z=np.arange(10) - 5)

    helper = StateAttributeLimitsHelper(state, 'layer', 'comp',
                                        'lower', 'higher',
                                        percentile='scale', log='log')

    helper.component_id = state.layer.id['x']

max_stars_repo_name: ejeschke/glue

max_issues_repo_name: ejeschke/glue

max_forks_repo_name: ejeschke/glue


text:
import cv2
import numpy as np

def overlay_transparent(mainimage, overlay, x, y):

    background  = mainimage.copy()

    background_width = background.shape[1]
    background_height = background.shape[0]

    maskBG = np.zeros([background_width, background_height, background.shape[2]], dtype = np.uint8)

    if x >= background_width or y >= background_height:
        return background

    h, w = overlay.shape[0], overlay.shape[1]

    if x + w > background_width:
        # w = background_width - x
        x = background_width - w
        overlay = overlay[:, :w]

    if y + h > background_height:
        # h = background_height - y
        y = background_height - h
        overlay = overlay[:h]

    if overlay.shape[2] < 4:
        overlay = np.concatenate(
            [
                overlay,
                np.ones((overlay.shape[0], overlay.shape[1], 1), dtype = overlay.dtype) * 255
            ],
            axis = 2,
        )

    overlay_image = overlay[..., :3]
    mask = overlay[..., 3:] / 255.0

    background[y:y+h, x:x+w] = (1.0 - mask) * background[y:y+h, x:x+w] + mask * overlay_image

    maskBG[y:y+h, x:x+w] = ((1.0 - mask) * maskBG[y:y+h, x:x+w] + mask * 255)

    return background, maskBG

from random import randint
import cv2
import numpy as np

path_FG_BG = 'Output/FG_BG'
path_mask_FG_BG = 'Output/mask_FG_BG'

quadrantsX  = [ 100, 0, 100, 0]
quadrantsY  = [ 100, 100, 0, 0]

def makeCombinations( backgound, foreground, opDir, imageNo):

  fg_img_tiny = cv2.resize(foreground, (foreground.shape[0], foreground.shape[1]), interpolation = cv2.INTER_AREA)
  for i in range(0,2):
    x = randint(0, 199 - (fg_img_tiny.shape[0]%199))
    y = randint(0, 199 - (fg_img_tiny.shape[1]%199))
    fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

    imageNo = imageNo + 1
    mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
    cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
    cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

    # cv2_imshow(fg_bg)
    # cv2_imshow(masked)

  fg_img_tiny = cv2.resize(foreground, (int(0.8*foreground.shape[0]), int(0.8*foreground.shape[1])), interpolation = cv2.INTER_AREA)
  for i in range(0,4):
    x = randint( quadrantsX[i%4], quadrantsX[i%4] + 99 - (fg_img_tiny.shape[0]%99) )
    y = randint( quadrantsY[i%4], quadrantsY[i%4] + 99 - (fg_img_tiny.shape[1]%99))
    fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

    imageNo = imageNo + 1
    mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
    cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
    cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

    # cv2_imshow(fg_bg)
    # cv2_imshow(masked)

  fg_img_tiny = cv2.resize(foreground, (int(0.6*foreground.shape[0]), int(0.6*foreground.shape[1])), interpolation = cv2.INTER_AREA)
  for i in range(0,5):
    x = randint( quadrantsX[i%4], quadrantsX[i%4] + 99 - (fg_img_tiny.shape[0]%99) )
    y = randint( quadrantsY[i%4], quadrantsY[i%4] + 99 - (fg_img_tiny.shape[1]%99) )
    fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

    imageNo = imageNo + 1
    mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
    cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
    cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

    # cv2_imshow(fg_bg)
    # cv2_imshow(masked)

  x = 100 - (int(fg_img_tiny.shape[0]/2))
  y = 100 - (int(fg_img_tiny.shape[1]/2))
  fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

  imageNo = imageNo + 1
  mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
  cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
  cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

  # cv2_imshow(fg_bg)
  # cv2_imshow(masked)

  fg_img_tiny = cv2.resize(foreground, (int(0.5*foreground.shape[0]), int(0.5*foreground.shape[1])), interpolation = cv2.INTER_AREA)
  for i in range(0,5):
    x = randint( quadrantsX[i%4], quadrantsX[i%4] + 99 - (fg_img_tiny.shape[0]%99))
    y = randint( quadrantsY[i%4], quadrantsY[i%4] + 99 - (fg_img_tiny.shape[1]%99))
    fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

    imageNo = imageNo + 1
    mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
    cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
    cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

    # cv2_imshow(fg_bg)
    # cv2_imshow(masked)

  x = 100 - (int(fg_img_tiny.shape[0]/2))
  y = 100 - (int(fg_img_tiny.shape[1]/2))
  fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

  imageNo = imageNo + 1
  mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
  cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
  cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

  # cv2_imshow(fg_bg)
  # cv2_imshow(masked)

  fg_img_tiny = cv2.resize(foreground, (int(0.3*foreground.shape[0]), int(0.3*foreground.shape[1])), interpolation = cv2.INTER_AREA)
  for i in range(0,2):
    x = randint( quadrantsX[i%4], quadrantsX[i%4] + 99 - (fg_img_tiny.shape[0]%99))
    y = randint( quadrantsY[i%4], quadrantsY[i%4] + 99 - (fg_img_tiny.shape[1]%99))
    fg_bg, masked = overlay_transparent(backgound, fg_img_tiny, x, y)

    imageNo = imageNo + 1
    mask_grey = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
    cv2.imwrite(opDir+'/overlay/'+str(imageNo)+".jpg", fg_bg)
    cv2.imwrite(opDir+'/mask/'+str(imageNo)+".jpg", mask_grey)

    # cv2_imshow(fg_bg)
    # cv2_imshow(masked)

from os import listdir
from DenseDepth.EVADepth import denseDepthModel
# from google.colab.patches import cv2_imshow
# import cv2
from matplotlib.pyplot import imshow
import numpy as np
from PIL import Image

%matplotlib inline
import os, errno

path_BG = 'bg'
path_FG = 'fg150'
bg_imageListDir = listdir(path_BG)
fg_imageListDir = listdir(path_FG)
print(len(bg_imageListDir))
for bg_image in bg_imageListDir:
  print(bg_image)
  outputDir = 'Dataset/'
  outputDir = outputDir + bg_image[:-4]
  bg_img = Image.open(path_BG+'/'+bg_image).resize((224,224), Image.ANTIALIAS)
  print(bg_img.size)
  # cv2_imshow(bg_img)
  for fg_image in fg_imageListDir:
    fg_img = Image.open(path_FG+'/'+fg_image).resize((224,224), Image.ANTIALIAS)
    break
  break
    # outputDir1 = outputDir + '/' + fg_image[:-4]
    # try:
    #     os.makedirs(outputDir1 + "/overlay")
    #     os.makedirs(outputDir1 + "/mask")
    #     os.makedirs(outputDir1 + "/depth")
    # except FileExistsError:
    #     pass

    # makeCombinations( bg_img, fg_img, outputDir1, 0)
    # #Flip foreground
    # fg_img_flip = cv2.flip(fg_img, 1)
    # makeCombinations( bg_img, fg_img_flip, outputDir1, 20)
    # denseDepthModel( model, outputDir1 + '/overlay/*.jpg', outputDir1 + '/depth/')


text:
"""Defines the forms for the ``jwql`` web app.

Django allows for an object-oriented model representation of forms for
users to provide input through HTTP POST methods. This module defines
all of the forms that are used across the various webpages used for the
JWQL application.

Authors
-------

    - Lauren Chambers
    - Johannes Sahlmann

Use
---

    This module is used within ``views.py`` as such:
    ::
        from .forms import FileSearchForm
        def view_function(request):
            form = FileSearchForm(request.POST or None)

            if request.method == 'POST':
                if form.is_valid():
                    # Process form input and redirect
                    return redirect(new_url)

            template = 'some_template.html'
            context = {'form': form, ...}
            return render(request, template, context)

References
----------
    For more information please see:
        ``https://docs.djangoproject.com/en/2.1/topics/forms/``

Dependencies
------------
    The user must have a configuration file named ``config.json``
    placed in the ``jwql/utils/`` directory.

"""
import glob
import os

from astropy.time import Time, TimeDelta
from django import forms
from django.shortcuts import redirect

from jwql.edb.edb_interface import is_valid_mnemonic
from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND
from jwql.utils.utils import get_config, filename_parser

FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')

class FileSearchForm(forms.Form):
    """Single-field form to search for a proposal or fileroot."""

    # Define search field
    search = forms.CharField(label='', max_length=500, required=True,
                             empty_value='Search')

    # Initialize attributes
    fileroot_dict = None
    search_type = None
    instrument = None

    def clean_search(self):
        """Validate the "search" field.

        Check that the input is either a proposal or fileroot, and one
        that matches files in the filesystem.

        Returns
        -------
        str
            The cleaned data input into the "search" field

        """
        # Get the cleaned search data
        search = self.cleaned_data['search']

        # Make sure the search is either a proposal or fileroot
        if len(search) == 5 and search.isnumeric():
            self.search_type = 'proposal'
        elif self._search_is_fileroot(search):
            self.search_type = 'fileroot'
        else:
            raise forms.ValidationError('Invalid search term {}. Please provide proposal number '
                                        'or file root.'.format(search))

        # If they searched for a proposal...
        if self.search_type == 'proposal':
            # See if there are any matching proposals and, if so, what
            # instrument they are for
            search_string = os.path.join(FILESYSTEM_DIR, 'jw{}'.format(search),
                                         '*{}*.fits'.format(search))
            all_files = glob.glob(search_string)
            if len(all_files) > 0:
                all_instruments = []
                for file in all_files:
                    instrument = filename_parser(file)['detector']
                    all_instruments.append(instrument[:3])
                if len(set(all_instruments)) > 1:
                    raise forms.ValidationError('Cannot return result for proposal with multiple '
                                                'instruments.')

                self.instrument = JWST_INSTRUMENT_NAMES_SHORTHAND[all_instruments[0]]
            else:
                raise forms.ValidationError('Proposal {} not in the filesystem.'.format(search))

        # If they searched for a fileroot...
        elif self.search_type == 'fileroot':
            # See if there are any matching fileroots and, if so, what
            # instrument they are for
            search_string = os.path.join(FILESYSTEM_DIR, search[:7], '{}*.fits'.format(search))
            all_files = glob.glob(search_string)

            if len(all_files) == 0:
                raise forms.ValidationError('Fileroot {} not in the filesystem.'.format(search))

            instrument = search.split('_')[-1][:3]
            self.instrument = JWST_INSTRUMENT_NAMES_SHORTHAND[instrument]

        return self.cleaned_data['search']

    def _search_is_fileroot(self, search):
        """Determine if a search value is formatted like a fileroot.

        Parameters
        ----------
        search : str
            The search term input by the user.

        Returns
        -------
        bool
            Is the search term formatted like a fileroot?

        """
        try:
            self.fileroot_dict = filename_parser(search)
            return True
        except ValueError:
            return False

    def redirect_to_files(self):
        """Determine where to redirect the web app based on user input.

        Returns
        -------
        HttpResponseRedirect object
            Outgoing redirect response sent to the webpage

        """
        # Process the data in form.cleaned_data as required
        search = self.cleaned_data['search']

        # If they searched for a proposal
        if self.search_type == 'proposal':
            return redirect('/{}/archive/{}'.format(self.instrument, search))

        # If they searched for a file root
        elif self.search_type == 'fileroot':
            return redirect('/{}/{}'.format(self.instrument, search))

class MnemonicSearchForm(forms.Form):
    """A single-field form to search for a mnemonic in the DMS EDB."""

    # Define search field
    search = forms.CharField(label='', max_length=500, required=True,
                             empty_value='Search', initial='SA_ZFGOUTFOV')

    # Initialize attributes
    search_type = None

    def clean_search(self):
        """Validate the "search" field.

        Check that the input is a valid mnemonic identifier.

        Returns
        -------
        str
            The cleaned data input into the "search" field

        """
        # Get the cleaned search data
        search = self.cleaned_data['search']

        # Make sure the search is a valid mnemonic identifier
        if is_valid_mnemonic(search):
            self.search_type = 'mnemonic'
        else:
            raise forms.ValidationError('Invalid search term {}. Please enter a valid DMS EDB '
                                        'mnemonic.'.format(search))

        return self.cleaned_data['search']

class MnemonicQueryForm(forms.Form):
    """A triple-field form to query mnemonic records in the DMS EDB."""

    production_mode = False

    if production_mode:
        # times for default query (one day one week ago)
        now = Time.now()
        delta_day = -7.
        range_day = 1.
        default_start_time = now + TimeDelta(delta_day, format='jd')
        default_end_time = now + TimeDelta(delta_day+range_day, format='jd')
    else:
        # example for testing
        default_start_time = Time('2019-01-16 00:00:00.000', format='iso')
        default_end_time = Time('2019-01-16 00:01:00.000', format='iso')

    default_mnemonic_identifier = 'IMIR_HK_ICE_SEC_VOLT4'

    # Define search fields
    search = forms.CharField(label='mnemonic', max_length=500, required=True,
                             initial=default_mnemonic_identifier, empty_value='Search',
                             help_text="Mnemonic identifier")

    start_time = forms.CharField(label='start', max_length=500, required=False,
                                 initial=default_start_time.iso, help_text="Start time")

    end_time = forms.CharField(label='end', max_length=500, required=False,
                               initial=default_end_time.iso, help_text="End time")

    # Initialize attributes
    search_type = None

    def clean_search(self):
        """Validate the "search" field.

        Check that the input is a valid mnemonic identifier.

        Returns
        -------
        str
            The cleaned data input into the "search" field

        """
        # Get the cleaned search data
        search = self.cleaned_data['search']

        if is_valid_mnemonic(search):
            self.search_type = 'mnemonic'
        else:
            raise forms.ValidationError('Invalid search term {}. Please enter a valid DMS EDB '
                                        'mnemonic.'.format(search))

        return self.cleaned_data['search']

    def clean_start_time(self):
        """Validate the start time.

        Returns
        -------
        str
           The cleaned data input into the start_time field

        """
        start_time = self.cleaned_data['start_time']
        try:
            Time(start_time, format='iso')
        except ValueError:
            raise forms.ValidationError('Invalid start time {}. Please enter a time in iso format, '
                                        'e.g. {}'.format(start_time, self.default_start_time))
        return self.cleaned_data['start_time']

    def clean_end_time(self):
        """Validate the end time.

        Returns
        -------
        str
           The cleaned data input into the end_time field

        """
        end_time = self.cleaned_data['end_time']
        try:
            Time(end_time, format='iso')
        except ValueError:
            raise forms.ValidationError('Invalid end time {}. Please enter a time in iso format, '
                                        'e.g. {}.'.format(end_time, self.default_end_time))

        if 'start_time' in self.cleaned_data.keys():
            # verify that end_time is later than start_time
            if self.cleaned_data['end_time'] <= self.cleaned_data['start_time']:
                raise forms.ValidationError('Invalid time inputs. End time is required to be after'
                                            ' Start time.')

        return self.cleaned_data['end_time']

class MnemonicExplorationForm(forms.Form):
    """A sextuple-field form to explore the EDB mnemonic inventory."""

    default_description = 'centroid data'

    # Define search fields
    description = forms.CharField(label='description', max_length=500, required=False,
                                  initial=default_description, help_text="Description")
    sql_data_type = forms.CharField(label='sqlDataType', max_length=500, required=False,
                                    help_text="sqlDataType")
    subsystem = forms.CharField(label='subsystem', max_length=500, required=False,
                                help_text="subsystem")
    tlm_identifier = forms.CharField(label='tlmIdentifier', max_length=500, required=False,
                                     help_text="Numerical ID (tlmIdentifier)")
    tlm_mnemonic = forms.CharField(label='tlmMnemonic', max_length=500, required=False,
                                   help_text="String ID (tlmMnemonic)")
    unit = forms.CharField(label='unit', max_length=500, required=False,
                           help_text="unit")

max_stars_repo_name: DanielKuebi/jwql

max_issues_repo_name: DanielKuebi/jwql

max_forks_repo_name: DanielKuebi/jwql

reason: from astropy


text:
"""Spectra retriever

Scrape site for new spectras that include wavelength and intensity
"""

from pathlib import Path

import numpy as np
from tqdm import tqdm

from .utils import check_folder, PERIODIC_TABLE, element_downloader

def spectra_retriever(
    output_folder: Path = Path("spectras2"),
    force_redownload: bool = False,
):
    """Downloads spectra.

    Downloads atomic spectra from nist.gov.

    Keyword Arguments:
        force_redownload {bool}
        output_folder {Path} -- saves the downloaded spectra in given folder.
            (default: {Path("spectras2")})
    """

    check_folder(output_folder)

    # Returns spectra, energies
    for elem in tqdm(PERIODIC_TABLE):

        filepath = output_folder / f"{elem[2]}.dat"

        if filepath.exists() and not force_redownload:
            tqdm.write(f"{filepath} already exist.")
            continue

        values = element_downloader(elem[2], return_fmt="dict")

        # Sorts the elements into an array
        elements = np.empty((len(values), 2), dtype=float)
        for i, val in enumerate(values):
            elements[i][0] = val["spectra"]
            elements[i][1] = val["intensity"]

        np.savetxt(filepath, elements)
        tqdm.write(f"Downloaded {elem[0]} and wrote to file: {filepath}")

max_stars_repo_name: hmvege/ElementSound

max_issues_repo_name: hmvege/ElementSound

max_forks_repo_name: hmvege/Elemental


text:
import random
import time
import copy
import os

import numpy as np

try:
    from keras_preprocessing.image import ImageDataGenerator
except ModuleNotFoundError:
    from tensorflow.keras.preprocessing.image import ImageDataGenerator
from keras_preprocessing.image.utils import (array_to_img, img_to_array)

from PIL import ImageEnhance

from .directory_iterator import ParallelDirectoryIterator

class ParallelImageDataGenerator(ImageDataGenerator):
    '''Generate batches of tensor image data with real-time data augmentation.

    The data will be looped over (in batches).

    This class works the exact same as the `Keras ImageDataGenerator <https://keras.io/preprocessing/image>`_
    except images are processed in the background using the standard Python
    `multiprocessing <https://docs.python.org/3.7/library/multiprocessing.html>`_ module.

    The can greatly improve training times as multiple CPU cores can process batch images in the background while
    training is done in the foreground on the GPU(s).
    (The standard Keras ImageDataGenerator module processes batch images then trains serially)

    From the outside, this module works the exact same as `Keras ImageDataGenerator <https://keras.io/preprocessing/image>`_.

    .. note::
       Since this uses the standard Python `multiprocessing <https://docs.python.org/3.7/library/multiprocessing.html>`_ module,
       the 'main' part of your Python script needs a:

    .. highlight:: python
    .. code-block:: python

       if __name__ == '__main__':
          ...

    to function correctly.

    It is also recommended to add something like:

    .. highlight:: python
    .. code-block:: python

       if current_process().name != 'MainProcess':
           # Disable the GPU and logging if this is a subprocess
           os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
           os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    to the top of the script to ensure that the GPU is not used in any subprocesses.

    Args:
        cores: The number of CPU cores to use for spawned image batch processes.
            This number can be either an integer, which specifies the exact number
            of CPU cores, or it can be a float < 1.0. The float is the percentage
            of CPU cores to use for processing.
            A large number of CPU cores will consume more system memory.

        debug: Use a ThreadPool rather than a Multiprocessing Pool,
            this allows for single-step debugging the processing function

        max_batches_pending: This is the number of processed batches to queue.
            A larger number can improving training times at the expense of
            increased system memory usage.

        validation_augmentation_enabled: If True, then augmentations will be applied to
            validation data. If False, then no augmentations will be applied to validation data.

        featurewise_center: Boolean.
            Set input mean to 0 over the dataset, feature-wise.

        samplewise_center: Boolean. Set each sample mean to 0.

        featurewise_std_normalization: Boolean.
            Divide inputs by std of the dataset, feature-wise.

        samplewise_std_normalization: Boolean. Divide each input by its std.

        zca_epsilon: epsilon for ZCA whitening. Default is 1e-6.

        zca_whitening: Boolean. Apply ZCA whitening.

        rotation_range: Int. Degree range for random rotations.

        width_shift_range: Float, 1-D array-like or int

            * float: fraction of total width, if < 1, or pixels if >= 1.
            * 1-D array-like: random elements from the array.
            * int: integer number of pixels from interval
              ``(-width_shift_range, +width_shift_range)``
            * With ``width_shift_range=2`` possible values
              are integers ``[-1, 0, +1]``,
              same as with ``width_shift_range=[-1, 0, +1]``,
              while with ``width_shift_range=1.0`` possible values are floats
              in the interval [-1.0, +1.0).

        height_shift_range: Float, 1-D array-like or int

            * float: fraction of total height, if < 1, or pixels if >= 1.
            * 1-D array-like: random elements from the array.
            * int: integer number of pixels from interval
              ``(-height_shift_range, +height_shift_range)``
            * With ``height_shift_range=2`` possible values
              are integers ``[-1, 0, +1]``,
              same as with ``height_shift_range=[-1, 0, +1]``,
              while with ``height_shift_range=1.0`` possible values are floats
              in the interval ``[-1.0, +1.0)``.

        brightness_range: Tuple or list of two floats. Range for picking
            a brightness shift value from.

        shear_range: Float. Shear Intensity
            (Shear angle in counter-clockwise direction in degrees)

        zoom_range: Float or [lower, upper]. Range for random zoom.
            If a float, ``[lower, upper] = [1-zoom_range, 1+zoom_range]``.

        channel_shift_range: Float. Range for random channel shifts.

        fill_mode: One of {"constant", "nearest", "reflect" or "wrap"}.
            Default is 'nearest'.
            Points outside the boundaries of the input are filled
            according to thegiven mode:

            * **constant** ``kkkkkkkk|abcd|kkkkkkkk (cval=k)``
            * **nearest**  ``aaaaaaaa|abcd|dddddddd``
            * **reflect**  ``abcddcba|abcd|dcbaabcd``
            * **wrap**  ``abcdabcd|abcd|abcdabcd``

        cval: Float or Int.
            Value used for points outside the boundaries
            when ``fill_mode = "constant"``.

        horizontal_flip: Boolean. Randomly flip inputs horizontally.

        vertical_flip: Boolean. Randomly flip inputs vertically.

        rescale: rescaling factor. Defaults to None.
            If None or 0, no rescaling is applied,
            otherwise we multiply the data by the value provided
            (after applying all other transformations).

        get_batch_function: function that should return the transformed batch.
            If this is omitted, then iterator.get_batches_of_transformed_samples() is used.
            NOTE: If this is supplied, then none of the other callbacks are used.
            This function should have the following signature:

            .. highlight:: python
            .. code-block:: python

               def get_batches_of_transformed_samples(
                  batch_index:int,
                  filenames:List[str],
                  classes:List[int],
                  params:ParallelProcessParams
               ) -> Tuple[int, Tuple[np.ndarray, np.ndarray]]:
                   ...
                   return batch_index, (batch_x, batch_y)

        noaug_preprocessing_function: function that will be applied on each input.
            The function will run after the image is resized but before it is augmented or standardized.
            The function should take at least two arguments and return the processed image:

            .. highlight:: python
            .. code-block:: python

               def my_processing_func(
                   params: ParallelProcessParams,
                   x : np.ndarray,
                   class_id: Optional[int],
                   filename: Optional[str],
                   batch_index: Optional[int],
                   batch_class_ids: Optional[List[int]],
                   batch_filenames: Optional[List[str]]
                ) -> np.ndarray:
                   ...
                   return processed_x

        preprocessing_function: function that will be applied on each input.
            The function will run after the image is resized (if ``interpolation != None``) and augmented but before it is standardized.
            The function should take at least two arguments and return the processed image:

            .. highlight:: python
            .. code-block:: python

               def my_processing_func(
                   params: ParallelProcessParams,
                   x : np.ndarray,
                   class_id: Optional[int],
                   filename: Optional[str],
                   batch_index: Optional[int],
                   batch_class_ids: Optional[List[int]],
                   batch_filenames: Optional[List[str]]
                ) -> np.ndarray:
                  ...
                  return processed_x

        validation_split: Float. Fraction of images reserved for validation
            (strictly between 0 and 1).

        dtype: Dtype to use for the generated arrays.

        brightness_range: Tuple of two floats. Control the brightness of an image.
            An enhancement factor of 0.0 gives a black image.
            A factor of 1.0 gives the original image.

        contrast_range: Tuple of two floats. Control the contrast of an image,
            similar to the contrast control on a TV set.
            An enhancement factor of 0.0 gives a solid grey image. A factor of 1.0 gives the original image.

        noise: List one one or more of the following:

            * **gauss** - Gaussian-distributed additive noise.
            * **poisson** - Poisson-distributed noise generated from the data.
            * **s&p** - Replaces random pixels with 0 or 1.
            * **speckle** - Multiplicative noise using out = image + n*image,where
                n is uniform noise with specified mean & variance.

            The noise type used will be randomly selected from the provided list per image

        random_transforms_enabled: Enable random data augmentations. Default True

        max_samples_per_class: The maximum number of samples to use for a given class.
            If ``-1`` then use all available samples. Default -1.

        disable_gpu_in_subprocesses: Disable GPU usage in spawned subprocesses, default: true

        batch_size: Generated batch size. This overrides the value given to flow_from_directory()
             Set to ``-1`` to set the batch size to be the number of samples

    '''
    def __init__(
        self, *,
        cores=0.25,
        debug=False,
        max_batches_pending=4,
        get_batch_function=None,
        preprocessing_function=None,
        noaug_preprocessing_function=None,
        validation_augmentation_enabled=True,
        contrast_range=None,
        noise=None,
        random_transforms_enabled=True,
        max_samples_per_class = -1,
        disable_gpu_in_subprocesses=True,
        save_to_dir=None,
        save_prefix=None,
        save_format=None,
        batch_size=None,
        **kwargs
    ):
        ImageDataGenerator.__init__(self, **_remove_preprocessing_function_arg(kwargs))

        self.cores = cores
        self.debug = debug
        self.random_transforms_enabled = random_transforms_enabled
        self.validation_augmentation_enabled = validation_augmentation_enabled
        self.max_batches_pending = max_batches_pending
        self.get_batch_function = get_batch_function
        self.parallel_preprocessing_function = preprocessing_function
        self.parallel_noaug_preprocessing_function = noaug_preprocessing_function
        self.contrast_range = contrast_range
        self.noise = noise
        self.max_samples_per_class = max_samples_per_class
        self.disable_gpu_in_subprocesses = disable_gpu_in_subprocesses
        self.save_to_dir = save_to_dir
        self.save_prefix = save_prefix
        self.save_format = save_format
        self.batch_size = batch_size

    def flow_from_directory(
        self,
        directory,
        target_size=(256, 256),
        color_mode='rgb',
        classes=None,
        class_mode='categorical',
        batch_size=32,
        batch_shape=None,
        shuffle=True,
        shuffle_index_dir=None,
        seed=None,
        save_to_dir=None,
        save_prefix=None,
        save_format='png',
        follow_links=False,
        subset=None,
        interpolation='bilinear',
        list_valid_filenames_in_directory_function=None,
    ):
        """Create the ParallelImageDataGenerator with the given dataset directory

        Takes the path to a directory & generates batches of augmented data.

        Args:

            directory: string, path to the target directory. It should contain one subdirectory per class.
                Any PNG, JPG, BMP, PPM or TIF images inside each of the subdirectories directory tree will be included in the generator.

            target_size: Tuple of integers (height, width), defaults to (256, 256).
                The dimensions to which all images found will be resized.

            color_mode: One of "grayscale", "rgb", "rgba". Default: "rgb".
                Whether the images will be converted to have 1, 3, or 4 channels.

            classes: Required, list of class subdirectories (e.g. ['dogs', 'cats']).

            class_mode: One of "categorical", "binary", "sparse", "input", or None.
                Default: "categorical". Determines the type of label arrays that are returned:

                * **categorical** will be 2D one-hot encoded labels,
                * **binary** will be 1D binary labels, "sparse" will be 1D integer labels,
                * **input** will be images identical to input images (mainly used to work with autoencoders).
                * **None** no labels are returned (the generator will only yield batches of image data, which is useful to use with model.predict()).

                Please note that in case of class_mode None, the data still needs to reside in a subdirectory of directory for it to work correctly.

            batch_size: Size of the batches of data (default: 32).

            batch_shape: Shape of the batches of data. If omitted, this defaults to (batch_size, target_size)

            shuffle: Whether to shuffle the data (default: True) If set to False, sorts the data in alphanumeric order.

            shuffle_index_dir: If given, the dataset directory will be shuffled the first time it is processed and
                and an index file containing the shuffled file names is generated at the directory specified
                by ``shuffle_index_dir``. The index file is reused to maintain the shuffled order for subsequent processing.
                If ``None``, then the dataset samples are sorted alphabetically and saved to an index file in the dataset directory.
                The alphabetical index file is used for subsequent processing.
                Default: ``None``

            seed: Optional random seed for shuffling and transformations.

            follow_links: Whether to follow symlinks inside class subdirectories (default: False).

            subset: Subset of data ("training" or "validation") if validation_split is set in ParallelAudioDataGenerator.

            interpolation: Interpolation method used to resample the image if the target size is different from that of the loaded image.
                Supported methods are ``none``, ``nearest``, ``bilinear``, ``bicubic``, ``lanczos``, ``box``, and ``hamming``.
                If ``none`` is used, then the image is **not automatically resized**.
                By default, ``bilinear`` is used.

            list_valid_filenames_in_directory_function: This is a custom function called for each class,
                that should return a list of valid file names for the given class.
                It has the following function signature:

                .. highlight:: python
                .. code-block:: python

                    def list_valid_filenames_in_directory(
                        base_directory:str,
                        search_class:str,
                        white_list_formats:List[str],
                        split:float,
                        follow_links:bool,
                        shuffle_index_directory:str
                    ) -> Tuple[str, List[str]]
                        ...
                        return search_class, filenames

            Returns:
                A DirectoryIterator yielding tuples of (x, y) where x is a numpy array containing a batch of images with
                shape (batch_size, *target_size, channels) and y is a numpy array of corresponding labels.

        """

        save_to_dir = self.save_to_dir or save_to_dir
        save_prefix = self.save_prefix or save_prefix
        save_format = self.save_format or save_format

        if batch_shape is None:
            if color_mode == 'rgba':
                image_shape = tuple(target_size) + (4,)
            elif self.color_mode == 'rgb':
                image_shape = tuple(target_size) + (3,)
            else:
                image_shape = tuple(target_size) + (1,)

            batch_shape = (batch_size,) + image_shape
        else:
            batch_shape = tuple(batch_shape)

        # If the batch_size was specified in the constructor
        # then that should override everything else
        if self.batch_size  is not None:
            batch_shape = (self.batch_size,) + batch_shape[1:]

        if save_to_dir:
            os.makedirs(save_to_dir, exist_ok=True)

        return ParallelDirectoryIterator(
            directory,
            self,
            target_size=target_size,
            color_mode=color_mode,
            classes=classes,
            class_mode=class_mode,
            dtype=self.dtype,
            data_format=self.data_format,
            batch_shape=batch_shape,
            shuffle=shuffle,
            shuffle_index_dir=shuffle_index_dir,
            seed=seed,
            save_to_dir=save_to_dir,
            save_prefix=save_prefix,
            save_format=save_format,
            follow_links=follow_links,
            subset=subset,
            interpolation=interpolation,
            cores=self.cores,
            debug = self.debug,
            max_batches_pending=self.max_batches_pending,
            get_batch_function=self.get_batch_function,
            preprocessing_function=self.parallel_preprocessing_function,
            noaug_preprocessing_function=self.parallel_noaug_preprocessing_function,
            list_valid_filenames_in_directory_function=list_valid_filenames_in_directory_function,
            max_samples_per_class=self.max_samples_per_class,
            disable_gpu_in_subprocesses=self.disable_gpu_in_subprocesses
        )

    def flow(
        self,
        x,
        y=None,
        batch_size=32,
        shuffle=True,
        sample_weight=None,
        seed=None,
        save_to_dir=None,
        save_prefix='',
        save_format='png',
        subset=None
    ):
        """Takes data & label arrays, generates batches of augmented data.

            Args:
                x: Input data. Numpy array of rank 4 or a tuple. If tuple, the first
                    element should contain the images and the second element another numpy
                    array or a list of numpy arrays that gets passed to the output without
                    any modifications. Can be used to feed the model miscellaneous data
                    along with the images. In case of grayscale data, the channels axis of
                    the image array should have value 1, in case of RGB data, it should
                    have value 3, and in case of RGBA data, it should have value 4.
                y: Labels.
                batch_size: Int (default: 32).
                shuffle: Boolean (default: True).
                sample_weight: Sample weights.
                seed: Int (default: None).
                save_to_dir: None or str (default: None). This allows you to optionally
                    specify a directory to which to save the augmented pictures being
                    generated (useful for visualizing what you are doing).
                save_prefix: Str (default: ``''``). Prefix to use for filenames of saved
                    pictures (only relevant if `save_to_dir` is set).
                save_format: one of "png", "jpeg", "bmp", "pdf", "ppm", "gif",
                    "tif", "jpg"
                    (only relevant if ``save_to_dir`` is set). Default: "png".
                subset: Subset of data (``"training"`` or ``"validation"``) if
                    ``validation_split`` is set in ``ImageDataGenerator``.

            Returns:

                An ``Iterator`` yielding tuples of ``(x, y)``
                    where ``x`` is a numpy array of image data
                    (in the case of a single image input) or a list
                    of numpy arrays (in the case with
                    additional inputs) and ``y`` is a numpy array
                    of corresponding labels. If 'sample_weight' is not None,
                    the yielded tuples are of the form ``(x, y, sample_weight)``.
                    If ``y`` is None, only the numpy array ``x`` is returned.

            Raises:
                ValueError: If the Value of the argument, ``subset`` is other than
                    "training" or "validation".
            """
        save_to_dir = self.save_to_dir or save_to_dir
        save_prefix = self.save_prefix or save_prefix
        save_format = self.save_format or save_format
        batch_size = self.batch_size  or batch_size

        if save_to_dir:
            os.makedirs(save_to_dir, exist_ok=True)

        return ImageDataGenerator.flow(
            self,
            x=x,
            y=y,
            batch_size=batch_size,
            shuffle=shuffle,
            sample_weight=sample_weight,
            seed=seed,
            save_to_dir=save_to_dir,
            save_prefix=save_prefix,
            save_format=save_format,
            subset=subset
        )

    @property
    def default_transform(self) -> dict:
        """Return default transformation parameters"""
        return copy.deepcopy({
            'theta': 0,
            'tx': 0,
            'ty': 0,
            'shear': 0,
            'zx' : 1,
            'zy' : 1,
            'flip_horizontal': False,
            'flip_vertical': False,
            'channel_shift_intensity': None,
            'brightness': None,
            'contrast' : None,
            'noise': None
        })

    def get_random_transform(self, img_shape, seed=None) -> dict:
        """Generate a random transformation"""
        if not self.random_transforms_enabled:
            return self.default_transform

        params = super().get_random_transform(img_shape)

        contrast = None
        if self.contrast_range:
            contrast = np.random.uniform(self.contrast_range[0], self.contrast_range[1])

        noise = None
        if self.noise:
            noise = random.choice(self.noise)

        params['contrast'] = contrast
        params['noise'] = noise

        return params

    def apply_transform(self, x, transform_parameters):
        """Apply the given transformation to the given image"""
        x = super().apply_transform(x, transform_parameters)
        if transform_parameters.get('contrast') is not None:
            contrast = transform_parameters.get('contrast')
            x = array_to_img(x)
            x = imgenhancer_Contrast = ImageEnhance.Contrast(x)
            x = imgenhancer_Contrast.enhance(contrast)
            x = img_to_array(x)

        if transform_parameters.get('noise') is not None:
            x = self._apply_noise(x, transform_parameters.get('noise'))

        return x

    def _apply_noise(self, x, noise):
        if noise == "gauss":
            row, col, ch = x.shape
            mean = 0
            var = 0.1
            sigma = var**0.5
            gauss = np.random.normal(mean, sigma, (row,col,ch))
            gauss = gauss.reshape(row,col,ch)
            x = x + gauss

        elif noise == "s&p":
            row, col, ch = x.shape
            s_vs_p = 0.5
            amount = 0.004
            out = np.copy(x)

            #if grayscale, ignore last dimension
            shape = x.shape
            if shape[-1] == 1: shape = shape[:-1]

            # Salt mode
            num_salt = np.ceil(amount * x.size * s_vs_p)
            coords = [np.random.randint(0, i - 1, int(num_salt)) for i in shape]
            out[tuple(coords)] = 255

            # Pepper mode
            num_pepper = np.ceil(amount* x.size * (1. - s_vs_p))
            coords = [np.random.randint(0, i - 1, int(num_pepper)) for i in shape]
            out[tuple(coords)] = 0
            x = out

        elif noise == "poisson":
            vals = len(np.unique(x))
            vals = 2 ** np.ceil(np.log2(vals))
            x = np.random.poisson(x * vals) / float(vals)

        elif noise =="speckle":
            row, col, ch = x.shape
            gauss = np.random.randn(row,col,ch)
            gauss = gauss.reshape(row,col,ch)
            x = x + x * gauss

        return x

def _remove_preprocessing_function_arg(kwargs):
    if 'preprocessing_function' in kwargs:
        del kwargs['preprocessing_function']
    return kwargs

max_stars_repo_name: SiliconLabs/mltk

max_issues_repo_name: SiliconLabs/mltk

max_forks_repo_name: sldriedler/mltk