Merge branch 'Alex' into dev

Former-commit-id: 4140981ad833e1ab4641328b0f7decb7d46d258e
This commit is contained in:
djfiume
2019-08-13 15:25:45 -04:00
73 changed files with 78279 additions and 357 deletions

View File

@@ -0,0 +1 @@
ISO-8859-1

Binary file not shown.

View File

@@ -0,0 +1 @@
GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

View File

@@ -0,0 +1 @@
c9901b634ba0b33dbccdcd5a064e8e31e60d1a5a

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@
ISO-8859-1

View File

@@ -0,0 +1 @@
0609796fd50b0082453c64c7db06e0954eab00de

View File

@@ -0,0 +1 @@
GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

View File

@@ -0,0 +1 @@
c9901b634ba0b33dbccdcd5a064e8e31e60d1a5a

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@
ISO-8859-1

View File

@@ -0,0 +1 @@
fc708e706c96962c4a3f02f6bc88b8620c423efc

View File

@@ -0,0 +1 @@
GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

View File

@@ -0,0 +1 @@
c9901b634ba0b33dbccdcd5a064e8e31e60d1a5a

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@
ISO-8859-1

View File

@@ -0,0 +1 @@
c65370bca653502d9b27310102141cb4352ebff9

View File

@@ -0,0 +1 @@
GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

View File

@@ -0,0 +1 @@
c9901b634ba0b33dbccdcd5a064e8e31e60d1a5a

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@
ISO-8859-1

View File

@@ -0,0 +1 @@
e8ed5ef79aaa1eb60bc707ef536fcbc88385ffba

View File

@@ -0,0 +1 @@
GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

View File

@@ -0,0 +1 @@
c9901b634ba0b33dbccdcd5a064e8e31e60d1a5a

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1 @@
ISO-8859-1

View File

@@ -0,0 +1 @@
895d242a486f1d8e5b78e8144096fc46337f88cf

View File

@@ -0,0 +1 @@
GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

View File

@@ -0,0 +1 @@
c9901b634ba0b33dbccdcd5a064e8e31e60d1a5a

View File

@@ -3,6 +3,7 @@
"population",
"power_supply",
"power_demand",
"waterdemand"
"waterdemand",
"rcp_climate"
]
}

View File

@@ -26,7 +26,7 @@ class Broker:
self.max_incstep = 50
self.initial_year = 2016
self.boot_timer = 60 # units: seconds
self.watchdog_timer = 30 # units: seconds
self.watchdog_timer = 60 # units: seconds
self.client = None
self.mongo_queue = Queue()
self.broker_id = 'broker'

View File

@@ -10,7 +10,12 @@ services:
waterdemand:
build: ../models/examples/waterdemand/
volumes:
- ../models/examples/waterdemand:/opt:ro
- ../models/examples/waterdemand:/opt:ro
rcp_climate:
build: ../models/examples/rcp_climate/
volumes:
- ../models/examples/rcp_climate:/opt:ro
power_supply:
build: ../models/examples/power_supply/
@@ -27,8 +32,6 @@ services:
container_name: mongodb
labels:
simon: "database"
volumes:
- ../data/db:/data/db
expose:
- "27017"
ports:

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"climate": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["climate"]
{
"type": "object",
"properties": {
"climate": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["climate"]
}

View File

@@ -1,4 +1,4 @@
{
"title": "empty schema",
"description": "matches everything"
}
{
"title": "empty schema",
"description": "matches everything"
}

View File

@@ -1,35 +1,35 @@
import glob
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
import pyhector
import json
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="hector", num_expected_inputs=num_input_schemas)
def configure(self, **kwargs):
self.rcp = kwargs['bootstrap']['rcp']
def increment(self, **kwargs):
if self.rcp == "rcp26":
print("rcp26")
pandas_df = pyhector.run(pyhector.rcp26)
else:
print("rcp85")
pandas_df = pyhector.run(pyhector.rcp85)
return {'climate': {'climate': {'data': json.loads(pandas_df["temperature.Tgav"].to_json()), 'granularity': 'global'}}}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()
import glob
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
import pyhector
import json
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="hector", num_expected_inputs=num_input_schemas)
def configure(self, **kwargs):
self.rcp = kwargs['bootstrap']['rcp']
def increment(self, **kwargs):
if self.rcp == "rcp26":
print("rcp26")
pandas_df = pyhector.run(pyhector.rcp26)
else:
print("rcp85")
pandas_df = pyhector.run(pyhector.rcp85)
return {'climate': {'climate': {'data': json.loads(pandas_df["temperature.Tgav"].to_json()), 'granularity': 'global'}}}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()

View File

@@ -1,12 +1,12 @@
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
}
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
}
}

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["population"]
}
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["population"]
}

View File

@@ -1,41 +1,41 @@
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 11 11:38:23 2018
@author: ponzodj1
"""
import json
from statsmodels.tsa.holtwinters import Holt
import pandas as pd
import numpy as np
def pop_sim(init_data):
try:
with open('updated_populations.json') as file:
data = json.loads([x for x in file][0])
except FileNotFoundError:
data = init_data
# Creates a temp dict, iterates through the loaded json data dict in the form
# of {county1_index: {2000: pop, 2001: pop, etc}, county2_index:...}
# applies Holt linear trend method to predict one year ahead
# outputted data is dict of {county_index: next_year_pop}
temp = {}
for key, county in data.items():
population = pd.Series(county)
fit1 = Holt(np.asarray(population)).fit(smoothing_level=0.7, smoothing_slope=0.3)
next_year = fit1.forecast(1)[0]
temp[key] = next_year
data[key][str(int(max(data[key].keys())) + 1)] = next_year
with open('updated_populations.json', 'w') as file:
file.write(json.dumps(data))
return temp
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 11 11:38:23 2018
@author: ponzodj1
"""
import json
from statsmodels.tsa.holtwinters import Holt
import pandas as pd
import numpy as np
def pop_sim(init_data):
try:
with open('updated_populations.json') as file:
data = json.loads([x for x in file][0])
except FileNotFoundError:
data = init_data
# Creates a temp dict, iterates through the loaded json data dict in the form
# of {county1_index: {2000: pop, 2001: pop, etc}, county2_index:...}
# applies Holt linear trend method to predict one year ahead
# outputted data is dict of {county_index: next_year_pop}
temp = {}
for key, county in data.items():
population = pd.Series(county)
fit1 = Holt(np.asarray(population)).fit(smoothing_level=0.7, smoothing_slope=0.3)
next_year = fit1.forecast(1)[0]
temp[key] = next_year
data[key][str(int(max(data[key].keys())) + 1)] = next_year
with open('updated_populations.json', 'w') as file:
file.write(json.dumps(data))
return temp

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["population"]
}
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["population"]
}

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"power_demand": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["power_demand"]
{
"type": "object",
"properties": {
"power_demand": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["power_demand"]
}

View File

@@ -1,40 +1,40 @@
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 11 14:10:24 2018
@author: ponzodj1
"""
import pandas as pd
def pow_dem_sim(pop,cons):
#sets baseline initialization if no data received
# Must receive data as dict of {county_id: current_population, ...}
# loads in static state values
# simply multiplies current pop by state consumption per capita
temp = {}
count = pd.DataFrame(pop,index=['pop'])
count = count.T
count.reset_index(inplace=True)
count['State'] = count['index'].apply(lambda x: x[:-3])
state_pops = count.groupby('State').sum().reset_index()
count = pd.merge(count,state_pops,on='State',how='left')
count['perc'] = count.apply(lambda x: x.pop_x / x.pop_y, axis=1)
cons_pc = pd.DataFrame(cons,index=['cons'])
cons_pc = cons_pc.T
count = pd.merge(count,cons_pc.reset_index(), left_on='State',right_on='index',how='left')
count['demand'] = count.apply(lambda x: (x.pop_y * x.cons) * x.perc,axis=1)
count = count[['index_x','demand']].set_index('index_x')
for index, row in count.iterrows():
temp[index] = row['demand']
return temp
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 11 14:10:24 2018
@author: ponzodj1
"""
import pandas as pd
def pow_dem_sim(pop,cons):
#sets baseline initialization if no data received
# Must receive data as dict of {county_id: current_population, ...}
# loads in static state values
# simply multiplies current pop by state consumption per capita
temp = {}
count = pd.DataFrame(pop,index=['pop'])
count = count.T
count.reset_index(inplace=True)
count['State'] = count['index'].apply(lambda x: x[:-3])
state_pops = count.groupby('State').sum().reset_index()
count = pd.merge(count,state_pops,on='State',how='left')
count['perc'] = count.apply(lambda x: x.pop_x / x.pop_y, axis=1)
cons_pc = pd.DataFrame(cons,index=['cons'])
cons_pc = cons_pc.T
count = pd.merge(count,cons_pc.reset_index(), left_on='State',right_on='index',how='left')
count['demand'] = count.apply(lambda x: (x.pop_y * x.cons) * x.perc,axis=1)
count = count[['index_x','demand']].set_index('index_x')
for index, row in count.iterrows():
temp[index] = row['demand']
return temp

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"power_demand": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["power_demand"]
{
"type": "object",
"properties": {
"power_demand": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["power_demand"]
}

View File

@@ -1,20 +1,20 @@
{
"type": "object",
"properties": {
"co2": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
},
"thermo_water": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["co2", "thermo_water"]
{
"type": "object",
"properties": {
"co2": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
},
"thermo_water": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["co2", "thermo_water"]
}

View File

@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 11 15:14:47 2018
@@ -50,4 +49,3 @@ def gen_sim(demand,prof):
#4. aggregate it to anything you want, you want it as tight as possible so probably countynerc
## will be easily able to do things and we will be happy the nerc is there, still being processed at powerplant but given a scaling factor

View File

@@ -0,0 +1,3 @@
FROM simon-model:latest
RUN pip3 install numpy
CMD ["python3", "/opt/src/inner_wrapper.py"]

View File

@@ -0,0 +1 @@
a01857114d3bfc51db5cecf202bc6b30bb03dcc3

View File

@@ -0,0 +1 @@
68c5cebf938d41fb8dd722a5da9ecaeece156baf

View File

@@ -0,0 +1 @@
a290d34112862e27112167db73ceabe6ff758a60

View File

@@ -0,0 +1,4 @@
{
"title": "empty schema",
"description": "matches everything"
}

View File

@@ -0,0 +1,23 @@
{
"type": "object",
"properties": {
"global_temp": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"type": "object",
"properties": {
"rcp": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["global_temp", "rcp"]
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,40 @@
import glob
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
from climate_model import temp_inc
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="rcp_climate", num_expected_inputs=num_input_schemas)
def configure(self, **kwargs):
self.raw_data = kwargs['rcp26data']
if 'rcp26data' in kwargs.keys():
self.mean_temp, self.climate_data = temp_inc(self.raw_data, self.incstep)
# elif 'rcp60data' in kwargs.keys():
# self.data = temp_inc(kwargs['rcp60data'], self.incstep)
# elif 'rcp85data' in kwargs.keys():
# self.data = temp_inc(kwargs['rcp85data'], self.incstep)
else:
print('rcp data not found')
def increment(self, **kwargs):
self.global_temp, self.climate_data = temp_inc(self.raw_data, self.incstep)
translated = self.translate(self.climate_data, 'climate', 'HUC8', self.model_id)
results = {'rcp_climate': {'global_temp': {'data': {'temp': self.global_temp}, 'granularity': 'global'}, 'rcp': {'data': translated, 'granularity': 'climate'}}}
return results, {}, {}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()

View File

@@ -1,2 +1,2 @@
FROM simon-model:latest
CMD ["python3", "/opt/src/inner_wrapper.py"]
FROM simon-model:latest
CMD ["python3", "/opt/src/inner_wrapper.py"]

View File

@@ -1,3 +1,3 @@
{
{
}

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"climate": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["climate"]
{
"type": "object",
"properties": {
"climate": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["climate"]
}

View File

@@ -1,4 +1,4 @@
{
"title": "empty schema",
"description": "matches everything"
}
{
"title": "empty schema",
"description": "matches everything"
}

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["population"]
}
{
"type": "object",
"properties": {
"population": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["population"]
}

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"water": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["water"]
}
{
"type": "object",
"properties": {
"water": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["water"]
}

View File

@@ -1,4 +1,4 @@
{
"title": "empty schema",
"description": "matches everything"
}
{
"title": "empty schema",
"description": "matches everything"
}

View File

@@ -1,13 +1,13 @@
{
"type": "object",
"properties": {
"water": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["water"]
}
{
"type": "object",
"properties": {
"water": {
"type": "object",
"properties":{
"data": {"type": "object"},
"granularity": {"type": "string"}
}
}
},
"required": ["water"]
}

View File

@@ -1,30 +1,30 @@
import time
import glob
from random import randint
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="water", num_expected_inputs=num_input_schemas)
def configure(self, **kwargs):
pass
def increment(self, **kwargs):
print("inner wrapper increment")
time.sleep(randint(1, 3))
return {'water': {'water': {'data': {}, 'granularity': 'county'}}}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()
import time
import glob
from random import randint
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="water", num_expected_inputs=num_input_schemas)
def configure(self, **kwargs):
pass
def increment(self, **kwargs):
print("inner wrapper increment")
time.sleep(randint(1, 3))
return {'water': {'water': {'data': {}, 'granularity': 'county'}}}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()

View File

@@ -1,8 +1,8 @@
# build from the base model image (required)
FROM simon-model:latest
# install dependencies
RUN pip3 install numpy pandas
# run the inner wrapper (required)
CMD ["python3", "/opt/src/inner_wrapper.py"]
# build from the base model image (required)
FROM simon-model:latest
# install dependencies
RUN pip3 install numpy pandas
# run the inner wrapper (required)
CMD ["python3", "/opt/src/inner_wrapper.py"]

View File

@@ -1,2 +1,2 @@
{
}
{
}

View File

@@ -1,4 +1,4 @@
{
"title": "empty schema",
"description": "matches everything"
}
{
"title": "empty schema",
"description": "matches everything"
}

View File

@@ -1,4 +1,4 @@
{
"title": "empty schema",
"description": "matches everything"
}
{
"title": "empty schema",
"description": "matches everything"
}

View File

@@ -1,27 +1,27 @@
import glob
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="unique_model_name", num_expected_inputs=num_input_schemas)
self.data = None
def configure(self, **kwargs):
self.data = kwargs['schema_name']
def increment(self, **kwargs):
return {'schema_name': {'data_variable_name': {'data': {}, 'granularity': 'county'}}}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()
import glob
import sys
sys.path.append('/')
from outer_wrapper import OuterWrapper
class InnerWrapper(OuterWrapper):
def __init__(self):
num_input_schemas = len(glob.glob("/opt/schemas/input/*.json"))
super().__init__(model_id="unique_model_name", num_expected_inputs=num_input_schemas)
self.data = None
def configure(self, **kwargs):
self.data = kwargs['schema_name']
def increment(self, **kwargs):
return {'schema_name': {'data_variable_name': {'data': {}, 'granularity': 'county'}}}
def main():
wrapper = InnerWrapper()
wrapper.run()
if __name__ == "__main__":
main()