Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Christian Fischer Pedersen
AIR
Commits
e1eb738f
Commit
e1eb738f
authored
Dec 10, 2021
by
Christian Marius Lillelund
Browse files
updated paths
parent
ef58f4a4
Pipeline
#98961
passed with stage
in 5 minutes and 17 seconds
Changes
7
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
ml/data/raw/2021/.gitkeep
0 → 100644
View file @
e1eb738f
ml/notebooks/Alarm.ipynb
View file @
e1eb738f
...
...
@@ -1763,7 +1763,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.
8
"
"version": "3.8.
11
"
},
"orig_nbformat": 4
},
...
...
ml/src/data/load_and_clean_data.py
View file @
e1eb738f
...
...
@@ -23,7 +23,6 @@ def main():
ss
=
loader
.
load_status_set
(
pt
.
PATHS_2021
[
2
],
pt
.
PATHS_2021
[
4
],
pt
.
RAW_DATA_DIR_2021
)
ic
=
loader
.
load_iso_classes
(
'isoall.txt'
,
pt
.
REFERENCES_DIR
)
cleaner2021
=
cleaner
.
Cleaner2021
()
...
...
ml/src/data/make_dataset_emb.py
View file @
e1eb738f
...
...
@@ -65,12 +65,12 @@ def main(ats_resolution: int = None):
target_name
=
target_settings
[
'target_name'
],
metrics
=
target_settings
[
'metrics'
],
batch_size
=
target_settings
[
'batch_size'
],
train_ratio
=
target_settings
[
'train_ratio'
],
epochs
=
target_settings
[
'num_epochs'
],
optimizer
=
target_settings
[
'optimizer'
],
network_layers
=
target_settings
[
'network_layers'
],
verbose
=
target_settings
[
'verbose'
],
model_path
=
model_path
)
train_ratio
=
target_settings
[
'train_ratio'
],
epochs
=
target_settings
[
'num_epochs'
],
optimizer
=
target_settings
[
'optimizer'
],
network_layers
=
target_settings
[
'network_layers'
],
verbose
=
target_settings
[
'verbose'
],
model_path
=
model_path
)
else
:
ats_enc
=
encode_dataframe
(
df
=
df_ats_to_enc
,
target_name
=
target_settings
[
'target_name'
],
...
...
ml/src/db/insert_data_into_db.py
View file @
e1eb738f
...
...
@@ -69,8 +69,8 @@ def main():
cursor
=
cnxn
.
cursor
()
# General data
ids
=
lis
t
()
for
index
,
row
in
df_general
.
iterrows
():
ids
=
dic
t
()
for
_
,
row
in
df_general
.
iterrows
():
cursor
.
execute
(
"INSERT INTO dbo.citizen (first_name,last_name,ssn,age,gender) values(?,?,?,?,?)"
,
row
[
'Fornavn'
],
row
[
'Efternavn'
],
...
...
@@ -78,72 +78,46 @@ def main():
row
[
'Alder (aktuel)'
],
row
[
'Køn'
])
cursor
.
execute
(
"SELECT @@IDENTITY AS ID;"
)
ids
.
append
((
cursor
.
fetchone
()[
0
],
row
[
'Borger Id'
]))
break
cnxn
.
commit
()
ids
[
row
[
'Borger Id'
]]
=
cursor
.
fetchone
()[
0
]
# Home care
for
index
,
row
in
df_home_care
.
iterrows
():
citizen_id
=
int
(
ids
[
index
][
0
])
citizen_guid
=
ids
[
index
][
1
]
citizen_rows
=
df_home_care
.
loc
[
df_home_care
[
'Borger Id'
]
==
citizen_guid
]
counter
=
0
for
_
,
row
in
citizen_rows
.
iterrows
():
cursor
.
execute
(
"INSERT INTO dbo.home_care (citizen_id,date,type,name,time_slot) values(?,?,?,?,?)"
,
citizen_id
,
row
[
'Dato'
],
row
[
'Paragraf'
],
row
[
'Ydelse navn'
],
row
[
'Bestilt tid (minutter)'
].
replace
(
","
,
"."
))
counter
=
counter
+
1
if
counter
>
4
:
break
break
cnxn
.
commit
()
for
_
,
row
in
df_home_care
.
iterrows
():
citizen_guid
=
row
[
'Borger Id'
]
citizen_id
=
ids
[
citizen_guid
]
cursor
.
execute
(
"INSERT INTO dbo.home_care (citizen_id,date,type,name,time_slot) values(?,?,?,?,?)"
,
citizen_id
,
row
[
'Dato'
],
row
[
'Paragraf'
],
row
[
'Ydelse navn'
],
row
[
'Bestilt tid (minutter)'
].
replace
(
","
,
"."
))
# Training
for
index
,
row
in
df_training
.
iterrows
():
citizen_id
=
int
(
ids
[
index
][
0
])
citizen_guid
=
ids
[
index
][
1
]
citizen_rows
=
df_training
.
loc
[
df_training
[
'Borger Id'
]
==
citizen_guid
]
counter
=
0
for
_
,
row
in
citizen_rows
.
iterrows
():
cursor
.
execute
(
"INSERT INTO dbo.training_plans (citizen_id,name,start_date,end_date,status,time_slot) values(?,?,?,?,?,?)"
,
citizen_id
,
row
[
'Ydelse navn'
],
row
[
'Forløbsdato start'
],
row
[
'Forløbsdato slut'
],
row
[
'Visiteret status'
],
row
[
'Bestilt tid (minutter)'
])
counter
=
counter
+
1
if
counter
>
4
:
break
break
cnxn
.
commit
()
for
_
,
row
in
df_training
.
iterrows
():
citizen_guid
=
row
[
'Borger Id'
]
citizen_id
=
ids
[
citizen_guid
]
cursor
.
execute
(
"INSERT INTO dbo.training_plans (citizen_id,name,start_date,end_date,status,time_slot) values(?,?,?,?,?,?)"
,
citizen_id
,
row
[
'Ydelse navn'
],
row
[
'Forløbsdato start'
],
row
[
'Forløbsdato slut'
],
row
[
'Visiteret status'
],
row
[
'Bestilt tid (minutter)'
])
# Ats
ats
=
pd
.
read_csv
(
Path
.
joinpath
(
pt
.
REFERENCES_DIR
,
'ats.csv'
),
converters
=
{
'ats_id'
:
str
})
df_ats
[
'Kategori ISO nummer'
]
=
df_ats
[
'Kategori ISO nummer'
].
apply
(
lambda
x
:
x
[:
6
])
df_ats
[
'Kategori ISO navn'
]
=
preprocessor
.
replace_cat_values
(
df_ats
[[
'Kategori ISO nummer'
]],
ats
)
for
index
,
row
in
df_ats
.
iterrows
():
citizen_id
=
int
(
ids
[
index
][
0
])
citizen_guid
=
ids
[
index
][
1
]
citizen_rows
=
df_ats
.
loc
[
df_ats
[
'Borger id'
]
==
citizen_guid
]
counter
=
0
for
_
,
row
in
citizen_rows
.
iterrows
():
cursor
.
execute
(
"INSERT INTO dbo.assistive_aids (citizen_id,name,iso,initiative_name,paragraph,lend_date) values(?,?,?,?,?,?)"
,
citizen_id
,
row
[
'Kategori ISO navn'
],
row
[
'Kategori ISO nummer'
],
row
[
'Indsats navn'
],
row
[
'Paragraf'
],
row
[
'Kørselsdato'
])
counter
=
counter
+
1
if
counter
>
4
:
break
break
for
_
,
row
in
df_ats
.
iterrows
():
citizen_guid
=
row
[
'Borger id'
]
citizen_id
=
ids
[
citizen_guid
]
cursor
.
execute
(
"INSERT INTO dbo.assistive_aids (citizen_id,name,iso,initiative_name,paragraph,lend_date) values(?,?,?,?,?,?)"
,
citizen_id
,
row
[
'Kategori ISO navn'
],
row
[
'Kategori ISO nummer'
],
row
[
'Indsats navn'
],
row
[
'Paragraf'
],
row
[
'Kørselsdato'
])
cnxn
.
commit
()
cursor
.
close
()
def
read_csv
(
file_path
,
file_name
,
conveters
=
None
,
...
...
ml/src/paths.py
View file @
e1eb738f
...
...
@@ -7,9 +7,9 @@ PATHS_2020 = ['borgere_hmi_Rasmus_BorgerId_Gender_BirthYear.xlsx',
PATHS_2021
=
[
'AA_Finish Uden 50 udvalgte_Rasmus_Details-CPR.xlsx'
,
'AA_Hjælpemidler minus 50_Rasmus_Details-CPR.xlsx'
,
'AA_
rw-aeh@aalborg.dk_Rasmus + d
etails
-
CPR.xlsx'
,
'AA_
DigiRehab_Aalborg+D
etails
-
CPR.xlsx'
,
'VB_AIR Hjælpemidler udtræk Total+ Details - CPR.xlsx'
,
'VB_D
rPatientData_Rasmus+d
etails
-
CPR
s
.xlsx'
,
'VB_D
igiRehab_Viborg+D
etails
-
CPR.xlsx'
,
'VB_Viborg Kommune - Servicekald + Details - CPR.xlsx'
]
ROOT_DIR
=
Path
(
__file__
).
absolute
().
parent
.
parent
...
...
ml/src/tools/raw_loader.py
View file @
e1eb738f
...
...
@@ -181,7 +181,7 @@ class RawLoader2021(BaseRawLoader2021):
:param file_path: path of file
:return: dataframe with loaded data
"""
index_col
=
'
BorgerID
'
index_col
=
'
Id
'
converters
=
{
index_col
:
str
}
# pass as string to avoid float conversion
infile
=
BytesIO
()
with
open
(
Path
.
joinpath
(
file_path
,
aalborg_file_name
),
'rb'
)
as
fd
:
...
...
@@ -231,7 +231,7 @@ class RawLoader2021(BaseRawLoader2021):
:param file_path: path of file
:return: dataframe with loaded data
"""
index_col
=
'
BorgerID
'
index_col
=
'
Id
'
converters
=
{
index_col
:
str
}
# pass as string to avoid float conversion
infile
=
BytesIO
()
with
open
(
Path
.
joinpath
(
file_path
,
aalborg_file_name
),
'rb'
)
as
fd
:
...
...
@@ -278,7 +278,7 @@ class RawLoader2021(BaseRawLoader2021):
:param file_path: path of file
:return: dataframe with loaded data
"""
index_col
=
'
BorgerID
'
index_col
=
'
Id
'
converters
=
{
index_col
:
str
}
# pass as string to avoid float conversion
infile
=
BytesIO
()
with
open
(
Path
.
joinpath
(
file_path
,
aalborg_file_name
),
'rb'
)
as
fd
:
...
...
@@ -338,7 +338,7 @@ class RawLoader2021(BaseRawLoader2021):
:param file_path: path of file
:return: dataframe with loaded data
"""
index_col
=
'
BorgerID
'
index_col
=
'
Id
'
converters
=
{
index_col
:
str
}
# pass as string to avoid float conversion
infile
=
BytesIO
()
with
open
(
Path
.
joinpath
(
file_path
,
aalborg_file_name
),
'rb'
)
as
fd
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment