Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor/station deps #32

Merged
merged 28 commits into from
May 20, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
6285d53
Update institutional header
dalonsoa May 18, 2022
f64cc96
Add menu contents to JSON file
dalonsoa May 18, 2022
14c0d44
Replace menu_struc for a function that reads json
dalonsoa May 18, 2022
b9c54c6
Simplify get user menu
dalonsoa May 18, 2022
be39ad6
Remove home/admin as it is not required
dalonsoa May 18, 2022
64902a1
Ignore import not used warning
dalonsoa May 18, 2022
2de05b2
Keep flake8 happy
dalonsoa May 18, 2022
3cf1984
Document home/context_processor.py
dalonsoa May 18, 2022
10a170d
Document and remove functions in home/functions.py
dalonsoa May 18, 2022
7db8490
Remove signals form home, as they are not really needed
dalonsoa May 18, 2022
7596411
Explicit imports from home.vies in url
dalonsoa May 18, 2022
efed27c
Clean up home/vies.py a bit
dalonsoa May 18, 2022
ae0b429
Keep flake8 happy with line length
dalonsoa May 18, 2022
74d620e
Run isort in home/
dalonsoa May 18, 2022
22c160c
Remove import home.signals from home/apps
dalonsoa May 18, 2022
9f9a4d3
Move apps to subdir
dandavies99 May 20, 2022
72aa6a3
exclude unused apps from isort
dandavies99 May 20, 2022
d4c75f5
keep isort happy after move
dandavies99 May 20, 2022
7a57d5b
Update imports
dandavies99 May 20, 2022
4a3d2a7
replace model name in other apps
dandavies99 May 20, 2022
63f7d54
Further replacements
dandavies99 May 20, 2022
1717550
more swaps
dandavies99 May 20, 2022
1d7415c
Keep isort happy
dandavies99 May 20, 2022
824b83b
few more translations
dandavies99 May 20, 2022
1d906e6
merge develop into refactor/station-deps
dandavies99 May 20, 2022
f1e869a
Merge branch 'develop' into refactor/station-deps
dandavies99 May 20, 2022
28bffe4
Apply suggestions from code review
dandavies99 May 20, 2022
c2b3992
fix merge marks
dandavies99 May 20, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cruce/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@
from django.db import models
from django.urls import reverse

from estacion.models import Estacion
from station.models import Station
from variable.models import Variable


class Cruce(models.Model):
cru_id = models.AutoField("Id", primary_key=True)
est_id = models.ForeignKey(
Estacion, models.SET_NULL, blank=True, null=True, verbose_name="Estación"
Station, models.SET_NULL, blank=True, null=True, verbose_name="Estación"
)
var_id = models.ForeignKey(
Variable, models.SET_NULL, blank=True, null=True, verbose_name="Variable"
Expand Down
2 changes: 1 addition & 1 deletion djangomain/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@

INSTALLED_APPS = [
"home.apps.HomeConfig",
"station.apps.EstacionConfig",
"station.apps.StationConfig",
"sensor.apps.SensorConfig",
"variable.apps.VariableConfig",
"cruce.apps.CruceConfig",
Expand Down
8 changes: 4 additions & 4 deletions formato/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from django.db import models
from django.urls import reverse

from estacion.models import Estacion
from station.models import Station
from variable.models import Variable


Expand Down Expand Up @@ -191,8 +191,8 @@ class Asociacion(models.Model):
for_id = models.ForeignKey(
Formato, models.SET_NULL, blank=True, null=True, verbose_name="Formato"
)
est_id = models.ForeignKey(
Estacion, models.SET_NULL, blank=True, null=True, verbose_name="Estación"
station_id = models.ForeignKey(
Station, models.SET_NULL, blank=True, null=True, verbose_name="Station"
)

def get_absolute_url(self):
Expand All @@ -201,6 +201,6 @@ def get_absolute_url(self):
class Meta:
ordering = ("aso_id",)
unique_together = (
"est_id",
"station_id",
"for_id",
)
6 changes: 3 additions & 3 deletions formato/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ class ClasificacionDetail(PermissionRequiredMixin, DetailView):
class AsociacionCreate(PermissionRequiredMixin, CreateView):
permission_required = "formato.add_asociacion"
model = Asociacion
fields = ["for_id", "est_id"]
fields = ["for_id", "station_id"]

def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
Expand All @@ -416,7 +416,7 @@ class AsociacionList(PermissionRequiredMixin, TemplateView):

def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
campos = ["aso_id", "for_id__for_nombre", "est_id__est_codigo"]
campos = ["aso_id", "for_id__for_nombre", "station_id__station_code"]
asociacion = Asociacion.objects.all().values_list(*campos)
context["asociacion"] = modelo_a_tabla_html(asociacion, col_extra=True)
return context
Expand All @@ -430,7 +430,7 @@ class AsociacionDetail(PermissionRequiredMixin, DetailView):
class AsociacionUpdate(PermissionRequiredMixin, UpdateView):
permission_required = "formato.change_asociacion"
model = Asociacion
fields = ["for_id", "est_id"]
fields = ["for_id", "station_id"]

def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
Expand Down
26 changes: 11 additions & 15 deletions home/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,17 @@ class LoginForm(forms.Form):
("sensor_marca.json", "SENSOR: Marca"),
("sensor_tipo.json", "SENSOR: Tipo"),
# ("datalogger_marca.json", "DATALOGGER: Marca"),
("estacion_tipo.json", "ESTACIÓN: Tipo"),
("estacion_ecosistema.json", "ESTACIÓN: Ecosistema"),
("estacion_pais.json", "ESTACIÓN: País"),
("estacion_region.json", "ESTACIÓN: Región"),
("estacion_sitio.json", "ESTACIÓN: Sitio"),
("estacion_cuenca.json", "ESTACIÓN: Cuenca"),
("estacion_sitiocuenca.json", "ESTACIÓN: Asociación Sitio-Cuenca"),
("estacion_socio.json", "ESTACIÓN: Socio"),
("estacion_estacion.json", "ESTACIÓN: Estación"),
("formato_asociacion.json", "FORMATO/ESTACION: Asociación de Formato con Estación"),
("cruce_cruce.json", "VARIABLE/ESTACION: Asociación Variable por Estación"),
# (
# "frecuencia_frecuencia.json",
# "FRECUENCIA/VARIABLE/ESTACION: Frecuencias por variable y por estación",
# ),
("station_stationtype.json", "STATION: StationType"),
("station_ecosystem.json", "STATION: Ecosystem"),
("station_country.json", "STATION: Country"),
("station_region.json", "STATION: Region"),
("station_place.json", "STATION: Place"),
("station_basin.json", "STATION: Basin"),
("station_placebasin.json", "STATION: Association Place-Basin"),
("station_institution.json", "STATION: Institution"),
("station_station.json", "STATION: Station"),
("formato_asociacion.json", "FORMATO/STATION: Association Format-Station"),
("cruce_cruce.json", "VARIABLE/STATION: Association Variable-Station"),
]


Expand Down
2 changes: 1 addition & 1 deletion home/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def get_anonymous_user() -> User:


def get_menu(user: User) -> str:
"""Generate the user menu in HTML, depending on its permissons.
"""Generate the user menu in HTML, depending on its permissions.

Args:
user (User): The user to generate the menu for.
Expand Down
22 changes: 1 addition & 21 deletions importacion/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,11 @@

from django import forms

from estacion.models import Estacion
from importacion.models import Importacion
from station.models import Station


class ImportacionForm(forms.Form):
imp_observacion = forms.CharField(
label="Observación", required=False, widget=forms.Textarea(attrs={"rows": "3"})
)


# class ImportacionSearchForm(forms.Form):
# estacion = forms.ModelChoiceField(required=False,
# queryset=Estacion.objects.order_by('est_id').all())
# fecha = forms.DateField(required=False, label="Fecha de Importación(dd/mm/yyyy)", input_formats=['%d/%m/%Y'])
# lista = []
#
# def filtrar(self, form):
# estacion = form.cleaned_data['estacion']
# fecha = form.cleaned_data['fecha']
# if estacion and fecha:
# lista = Importacion.objects.filter(est_id=estacion).filter(imp_fecha__date=fecha)
# elif estacion is None and fecha:
# lista = Importacion.objects.filter(imp_fecha__date=fecha)
# elif fecha is None and estacion:
# lista = Importacion.objects.filter(est_id=estacion)
# else:
# lista = Importacion.objects.all()
# return lista
74 changes: 37 additions & 37 deletions importacion/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,16 @@ def validar_fechas(importacion):
fecha_ini = importacion.imp_fecha_ini
fecha_fin = importacion.imp_fecha_fin
for_id_id = importacion.for_id_id
estacion = importacion.est_id
station = importacion.station_id
clasificacion = list(Clasificacion.objects.filter(for_id=for_id_id))

sobrescribe = False
result = []
for fila in clasificacion:
var_id = str(fila.var_id.var_id)
est_id = str(estacion.est_id)
ultima_fecha = ultima_fecha_cargada(est_id, var_id)
existe = existe_fechas(fecha_ini, fecha_fin, est_id, var_id)
station_id = str(station.station_id)
ultima_fecha = ultima_fecha_cargada(station_id, var_id)
existe = existe_fechas(fecha_ini, fecha_fin, station_id, var_id)
sobrescribe = sobrescribe or existe
resumen = {
"var_id": fila.var_id.var_id,
Expand All @@ -82,10 +82,10 @@ def validar_fechas(importacion):
return result, sobrescribe


def ultima_fecha_cargada(est_id, var_id):
def ultima_fecha_cargada(station_id, var_id):
print("ultima_fecha: " + str(time.ctime()))
sql = "SELECT fecha FROM medicion_var" + str(int(var_id)) + "medicion "
sql += " WHERE estacion_id=" + str(int(est_id))
sql += " WHERE station_id=" + str(int(station_id))
sql += " ORDER BY fecha DESC LIMIT 1"
with connection.cursor() as cursor:
cursor.execute(sql)
Expand All @@ -97,12 +97,12 @@ def ultima_fecha_cargada(est_id, var_id):
return informacion


def existe_fechas(ini, fin, est_id, var_id):
def existe_fechas(ini, fin, station_id, var_id):
sql = "SELECT id FROM medicion_var" + str(var_id) + "medicion "
sql += " WHERE fecha >= %s AND fecha <= %s AND estacion_id = %s "
sql += " WHERE fecha >= %s AND fecha <= %s AND station_id = %s "
sql += " LIMIT 1;"
query = globals()["Var" + str(var_id) + "Medicion"].objects.raw(
sql, (ini, fin, est_id)
sql, (ini, fin, station_id)
)
consulta = list(query)
if len(consulta) > 0:
Expand Down Expand Up @@ -249,40 +249,40 @@ def verificar_fechahora(fechahora, formatofechahora):
def guardar_datos__temp_a_final(imp_id, form):
importaciontemp = ImportacionTemp.objects.get(imp_id=imp_id)
formato = importaciontemp.for_id
estacion = importaciontemp.est_id
station = importaciontemp.station_id
ruta = str(BASE_DIR) + "/media/" + str(importaciontemp.imp_archivo)

datos = construir_matriz(ruta, formato, estacion)
datos = construir_matriz(ruta, formato, station)
for var_id, tabla in datos.items():
tabla = tabla.where((pd.notnull(tabla)), None)
data = list(tabla.itertuples(index=False, name=None))
sql = """
WITH
data AS (
SELECT DISTINCT u.fecha, u.valor, u.estacion_id
FROM unnest(%s::fecha__valor__estacion_id[]) u
SELECT DISTINCT u.fecha, u.valor, u.station_id
FROM unnest(%s::fecha__valor__station_id[]) u
ORDER BY u.fecha ASC
),
eliminar AS (
DELETE FROM medicion_var1medicion
WHERE estacion_id = (SELECT d.estacion_id FROM data d LIMIT 1)
WHERE station_id = (SELECT d.station_id FROM data d LIMIT 1)
AND fecha >= (SELECT d.fecha FROM data d ORDER BY d.fecha ASC LIMIT 1)
AND fecha <= (SELECT d.fecha FROM data d ORDER BY d.fecha DESC LIMIT 1)
returning *
)
INSERT INTO medicion_var1medicion(fecha, valor, estacion_id)
SELECT d.fecha, d.valor, d.estacion_id
INSERT INTO medicion_var1medicion(fecha, valor, station_id)
SELECT d.fecha, d.valor, d.station_id
FROM data d
;
"""
sql = sql.replace("var1", "var" + str(var_id))
sql = sql.replace(
"u.fecha, u.valor, u.estacion_id", "u." + ", u.".join(tabla.columns)
"u.fecha, u.valor, u.station_id", "u." + ", u.".join(tabla.columns)
)
sql = sql.replace("fecha__valor__estacion_id", "__".join(tabla.columns))
sql = sql.replace("fecha, valor, estacion_id", ", ".join(tabla.columns))
sql = sql.replace("fecha__valor__station_id", "__".join(tabla.columns))
sql = sql.replace("fecha, valor, station_id", ", ".join(tabla.columns))
sql = sql.replace(
"d.fecha, d.valor, d.estacion_id", "d." + ", d.".join(tabla.columns)
"d.fecha, d.valor, d.station_id", "d." + ", d.".join(tabla.columns)
)

with connection.cursor() as cursor:
Expand All @@ -297,7 +297,7 @@ def guardar_datos__temp_a_final(imp_id, form):
ruta_final_full = str(BASE_DIR) + "/media/" + ruta_final
shutil.copy(ruta, ruta_final_full)
importacion = Importacion(
est_id=importaciontemp.est_id,
station_id=importaciontemp.station_id,
for_id=importaciontemp.for_id,
imp_fecha=importaciontemp.imp_fecha,
imp_fecha_ini=importaciontemp.imp_fecha_ini,
Expand All @@ -314,7 +314,7 @@ def guardar_datos__temp_a_final(imp_id, form):
return importacion.imp_id


def construir_matriz(matriz_src, formato, estacion):
def construir_matriz(matriz_src, formato, station):
# TODO : Eliminar validar_datalogger, validar acumulado
# determinar si debemos restar 5 horas a la fecha del archivo
# cambiar_fecha = validar_datalogger(formato.mar_id)
Expand Down Expand Up @@ -421,7 +421,7 @@ def construir_matriz(matriz_src, formato, estacion):
datos = datos.dropna()
if var.resolucion:
datos["valor"] = datos["valor"] * float(var.resolucion)
datos["estacion_id"] = estacion.est_id
datos["station_id"] = station.station_id
datos_variables[var.var_id_id] = datos
return datos_variables

Expand Down Expand Up @@ -451,7 +451,7 @@ def numero_coma_decimal(val_str):

def insertar_nivel_regleta(importacion, nivelregleta):
nivelagua_mediciones = Var11Medicion.objects.filter(
estacion_id=importacion.est_id_id, fecha=importacion.imp_fecha_fin
station_id=importacion.station_id_id, fecha=importacion.imp_fecha_fin
)
nivelagua = None
for i in nivelagua_mediciones:
Expand All @@ -463,7 +463,7 @@ def insertar_nivel_regleta(importacion, nivelregleta):
except:
return False
Var14Medicion(
estacion_id=importacion.est_id_id,
station_id=importacion.station_id_id,
fecha_importacion=importacion.imp_fecha,
fecha_inicio=importacion.imp_fecha_ini,
fecha=importacion.imp_fecha_fin,
Expand All @@ -474,18 +474,18 @@ def insertar_nivel_regleta(importacion, nivelregleta):
).save()


# consultar formatos por datalogger y estacion
def consultar_formatos(estacion):
asociacion = list(Asociacion.objects.filter(est_id=estacion))
# consultar formatos por datalogger y station
def consultar_formatos(station):
asociacion = list(Asociacion.objects.filter(station_id=station))
lista = {}
for item in asociacion:
lista[item.for_id.for_id] = item.for_id.for_nombre
return lista


#
# def procesar_archivo_automatico(archivo, formato, estacion):
# datos = construir_matriz(archivo, formato, estacion)
# def procesar_archivo_automatico(archivo, formato, station):
# datos = construir_matriz(archivo, formato, station)
# return datos
#
#
Expand All @@ -507,9 +507,9 @@ def consultar_formatos(estacion):
# return estado
#
#
# # def guardar_vacios(informacion, estacion, observacion, fecha_archivo):
# # def guardar_vacios(informacion, station, observacion, fecha_archivo):
# # variable = Variable.objects.get(var_id=informacion.get('var_id'))
# # vacio = Vacios(est_id=estacion, var_id=variable,
# # vacio = Vacios(station_id=station, var_id=variable,
# # vac_fecha_ini=informacion.get('ultima_fecha').date(),
# # vac_hora_ini=informacion.get('ultima_fecha').time(),
# # vac_fecha_fin=fecha_archivo.date(),
Expand All @@ -528,12 +528,12 @@ def consultar_formatos(estacion):
# fec_fin = str(fecha_fin)
# year_ini = fecha_ini.strftime('%Y')
# year_fin = fecha_fin.strftime('%Y')
# est_id = str(importacion.est_id.est_id)
# station_id = str(importacion.station_id.station_id)
# var_cod = informacion.get('var_cod')
# var_id = informacion.get('var_id')
# if year_ini == year_fin:
# sql = 'DELETE FROM medicion_var' + str(var_id) + 'medicion '
# sql += ' WHERE estacion_id=' + str(est_id)
# sql += ' WHERE station_id=' + str(station_id)
# sql += ' AND fecha>=\'' + fec_ini + '\''
# sql += ' AND fecha<=\'' + fec_fin + '\''
# with connection.cursor() as cursor:
Expand All @@ -543,17 +543,17 @@ def consultar_formatos(estacion):
# for year in range_year:
# if str(year) == year_ini:
# sql = 'DELETE FROM medicion_var' + str(var_id) + 'medicion '
# sql += ' WHERE estacion_id=' + str(est_id)
# sql += ' WHERE station_id=' + str(station_id)
# sql += ' AND fecha>=\'' + fec_ini + '\''
#
# elif str(year) == year_fin:
# sql = 'DELETE FROM medicion_var' + str(var_id) + 'medicion '
# sql += ' WHERE estacion_id=' + str(est_id)
# sql += ' WHERE station_id=' + str(station_id)
# sql += ' AND fecha<=\'' + fec_fin + '\''
#
# else:
# sql = 'DELETE FROM medicion_var' + str(var_id) + 'medicion '
# sql += ' WHERE estacion_id=' + str(est_id)
# sql += ' WHERE station_id=' + str(station_id)
# with connection.cursor() as cursor:
# cursor.execute(sql)
#
Expand Down
6 changes: 3 additions & 3 deletions importacion/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@
from django.urls import reverse
from django.utils import timezone

from estacion.models import Estacion
from formato.models import Formato
from station.models import Station


class ImportacionBase(models.Model):
imp_id = models.AutoField("Id", primary_key=True)
est_id = models.ForeignKey(
Estacion, models.SET_NULL, blank=True, null=True, verbose_name="Estación"
station_id = models.ForeignKey(
Station, models.SET_NULL, blank=True, null=True, verbose_name="Station"
)
for_id = models.ForeignKey(
Formato, models.SET_NULL, blank=True, null=True, verbose_name="Formato"
Expand Down
Loading