Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Christian Fischer Pedersen
AIR
Commits
b180b856
Commit
b180b856
authored
Dec 22, 2021
by
Christian Marius Lillelund
Browse files
improved api alarm endpoint
parent
03c0091e
Changes
13
Hide whitespace changes
Inline
Side-by-side
ml/src/analysis/evaluate_balance.py
View file @
b180b856
...
...
@@ -70,4 +70,4 @@ def main():
writer
.
writerow
(
data
)
if
__name__
==
'__main__'
:
main
()
\ No newline at end of file
main
()
ml/src/analysis/evaluate_gender_bias.py
View file @
b180b856
...
...
@@ -260,4 +260,4 @@ def main():
plt
.
show
()
if
__name__
==
"__main__"
:
main
()
\ No newline at end of file
main
()
ml/src/analysis/evaluate_preprocessing.py
View file @
b180b856
...
...
@@ -136,4 +136,4 @@ def main():
writer
.
writerow
(
data
)
if
__name__
==
'__main__'
:
main
()
\ No newline at end of file
main
()
ml/src/analysis/evaluate_survival_case.py
View file @
b180b856
...
...
@@ -8,8 +8,8 @@ import paths as pt
from
tools
import
data_loader
from
utility.config
import
load_config
from
utility.data
import
write_csv
from
sksurv.ensemble
import
GradientBoostingSurvivalAnalysis
from
sklearn.model_selection
import
KFold
from
sksurv.ensemble
import
GradientBoostingSurvivalAnalysis
from
sksurv.metrics
import
(
concordance_index_censored
,
concordance_index_ipcw
,
integrated_brier_score
)
...
...
ml/src/analysis/find_best_ats_resolution.py
View file @
b180b856
...
...
@@ -113,4 +113,4 @@ def main():
bbox_inches
=
"tight"
)
if
__name__
==
"__main__"
:
main
()
\ No newline at end of file
main
()
ml/src/api/main.py
View file @
b180b856
...
...
@@ -11,15 +11,16 @@ import csv
import
joblib
import
yaml
import
pandas
as
pd
import
numpy
as
np
from
typing
import
Optional
from
fastapi
import
Depends
,
FastAPI
,
HTTPException
,
Request
from
fastapi.responses
import
JSONResponse
from
fastapi_jwt_auth
import
AuthJWT
from
fastapi_jwt_auth.exceptions
import
AuthJWTException
from
fastapi.security
import
HTTPBearer
,
HTTPAuthorizationCredentials
from
fastapi
import
Request
,
HTTPException
from
sksurv.ensemble
import
GradientBoostingSurvivalAnalysis
from
xgboost
import
XGBClassifier
from
decimal
import
Decimal
app
=
FastAPI
(
title
=
'AIR API'
,
version
=
'1.0'
,
description
=
'An API that classifies citizens based on data'
)
...
...
@@ -88,7 +89,7 @@ class TrainingOutputData(pydantic.BaseModel):
@
app
.
get
(
'/'
)
def
index
():
return
{
'message'
:
f
'AIR API v. 0.1'
}
return
{
'message'
:
'AIR API v. 0.1'
}
@
app
.
get
(
'/user'
,
dependencies
=
[
Depends
(
JWTBearer
())],
tags
=
[
"login"
])
def
user
(
Authorize
:
AuthJWT
=
Depends
()):
...
...
@@ -139,10 +140,12 @@ def predict_alarm(incoming_data: InputData):
df_for_alarm
=
add_embedding
(
df
.
copy
(),
'alarm'
,
ats_resolution
)
surv_func
=
alarm_model
.
predict_survival_function
(
df_for_alarm
)
event_times
=
[
int
(
x
)
for
x
in
surv_func
[
0
].
x
]
surv_probs
=
[
float
(
x
)
for
x
in
surv_func
[
0
].
y
]
prob_after_one_year
=
1
-
surv_probs
[
365
*
5
]
event_times
=
list
(
range
(
1
,
13
))
surv_probs
=
np
.
array
([
float
(
x
)
for
x
in
surv_func
[
0
].
y
][:
360
])
surv_probs
=
np
.
mean
(
surv_probs
.
reshape
(
-
1
,
30
),
axis
=
1
)
surv_probs
=
list
(
map
((
lambda
x
:
float
(
round
(
100
*
Decimal
(
x
),
1
))),
surv_probs
))
prob_after_one_year
=
100
-
surv_probs
[
-
1
]
alarm_arguments
=
generate_alarm_arguments
(
df
,
ats_resolution
,
prob_after_one_year
)
return
{
...
...
@@ -159,8 +162,8 @@ def predict_training(incoming_data: InputData):
data
=
validate_data
(
incoming_data
)
df
=
prepare_data
(
data
,
ats_resolution
)
complete_model
=
read_xgb_model
(
f
'complete_xgboost.joblib'
)
compliance_model
=
read_xgb_model
(
f
'compliance_xgboost.joblib'
)
complete_model
=
read_xgb_model
(
'complete_xgboost.joblib'
)
compliance_model
=
read_xgb_model
(
'compliance_xgboost.joblib'
)
df_for_complete
=
add_embedding
(
df
.
copy
(),
'complete'
,
ats_resolution
)
df_for_compliance
=
add_embedding
(
df
.
copy
(),
'compliance'
,
ats_resolution
)
...
...
@@ -264,7 +267,7 @@ def generate_alarm_arguments(df: pd.DataFrame,
arguments
.
append
(
loan_period_argument
)
arguments
.
append
(
"får efter et år en nødalarm"
)
arguments
.
append
(
f
"med
{
round
(
prob_after_one_year
*
100
,
1
)
}
% sandsynlighed"
)
arguments
.
append
(
f
"med
{
round
(
prob_after_one_year
,
1
)
}
% sandsynlighed"
)
return
arguments
def
load_settings
(
file_name
):
...
...
ml/src/data/make_dataset_count.py
View file @
b180b856
...
...
@@ -68,4 +68,4 @@ def main():
write_csv
(
df
,
file_path
,
file_name
)
if
__name__
==
"__main__"
:
main
()
\ No newline at end of file
main
()
ml/src/data/make_dataset_ohe.py
View file @
b180b856
...
...
@@ -51,4 +51,4 @@ def main():
write_csv
(
df
,
file_path
,
file_name
)
if
__name__
==
"__main__"
:
main
()
\ No newline at end of file
main
()
ml/src/data/make_dataset_ordinal.py
View file @
b180b856
...
...
@@ -61,4 +61,4 @@ def main():
write_csv
(
df
,
file_path
,
file_name
)
if
__name__
==
"__main__"
:
main
()
\ No newline at end of file
main
()
ml/src/model/train_alarm_model.py
View file @
b180b856
...
...
@@ -31,7 +31,7 @@ def main():
model
.
fit
(
X
,
y
)
file_path
=
pt
.
MODELS_DIR
file_name
=
f
'alarm_gradboost.joblib'
file_name
=
'alarm_gradboost.joblib'
with
open
(
Path
.
joinpath
(
file_path
,
file_name
),
'wb'
)
as
fd
:
outfile
=
BytesIO
()
file_writer
.
write_joblib
(
model
,
outfile
)
...
...
ml/src/model/train_complete_model.py
View file @
b180b856
...
...
@@ -38,7 +38,7 @@ def main():
model
.
fit
(
X
,
y
)
file_path
=
pt
.
MODELS_DIR
file_name
=
f
'complete_xgboost.joblib'
file_name
=
'complete_xgboost.joblib'
with
open
(
Path
.
joinpath
(
file_path
,
file_name
),
'wb'
)
as
fd
:
outfile
=
BytesIO
()
file_writer
.
write_joblib
(
model
,
outfile
)
...
...
ml/src/model/train_compliance_model.py
View file @
b180b856
...
...
@@ -38,7 +38,7 @@ def main():
model
.
fit
(
X
,
y
)
file_path
=
pt
.
MODELS_DIR
file_name
=
f
'compliance_xgboost.joblib'
file_name
=
'compliance_xgboost.joblib'
with
open
(
Path
.
joinpath
(
file_path
,
file_name
),
'wb'
)
as
fd
:
outfile
=
BytesIO
()
file_writer
.
write_joblib
(
model
,
outfile
)
...
...
ml/src/tools/data_loader.py
View file @
b180b856
...
...
@@ -8,10 +8,11 @@ from abc import ABC, abstractmethod
from
sklearn.model_selection
import
train_test_split
from
typing
import
Tuple
,
List
from
pathlib
import
Path
from
io
import
BytesIO
,
StringIO
from
io
import
StringIO
import
shutil
from
tools
import
file_reader
,
preprocessor
import
numpy
as
np
import
pandas
as
pd
class
BaseDataLoader
(
ABC
):
"""
...
...
@@ -20,22 +21,22 @@ class BaseDataLoader(ABC):
def
__init__
(
self
,
file_path
:
Path
,
file_name
:
str
,
settings
,
converters
=
None
):
settings
:
dict
,
converters
:
dict
=
None
):
"""Initilizer method that takes a file path, file name,
settings and optionally a converter"""
self
.
X
=
None
self
.
y
=
None
self
.
file_path
=
file_path
self
.
file_name
=
file_name
self
.
settings
=
settings
self
.
converters
=
converters
self
.
X
:
pd
.
DataFrame
=
None
self
.
y
:
np
.
ndarray
=
None
self
.
file_path
:
Path
=
file_path
self
.
file_name
:
str
=
file_name
self
.
settings
:
dict
=
settings
self
.
converters
:
dict
=
converters
@
abstractmethod
def
load_data
(
self
):
def
load_data
(
self
)
->
None
:
"""Loads the data from a data set at startup"""
def
get_data
(
self
):
def
get_data
(
self
)
->
Tuple
[
pd
.
DataFrame
,
np
.
ndarray
]
:
"""
This method returns the features and targets
:return: X and y
...
...
@@ -104,8 +105,8 @@ class CompleteDataLoader(BaseDataLoader):
df
=
file_reader
.
read_csv
(
infile
,
converters
=
self
.
converters
)
X
=
df
.
drop
([
'Complete'
],
axis
=
1
)
y
=
df
[
'Complete'
]
self
.
X
=
X
self
.
y
=
y
self
.
X
=
pd
.
DataFrame
(
X
)
self
.
y
=
np
.
array
(
y
)
return
self
class
ComplianceDataLoader
(
BaseDataLoader
):
...
...
@@ -121,8 +122,8 @@ class ComplianceDataLoader(BaseDataLoader):
df
=
file_reader
.
read_csv
(
infile
,
converters
=
self
.
converters
)
X
=
df
.
drop
([
'Compliance'
],
axis
=
1
)
y
=
df
[
'Compliance'
]
self
.
X
=
X
self
.
y
=
y
self
.
X
=
pd
.
DataFrame
(
X
)
self
.
y
=
np
.
array
(
y
)
return
self
class
FallDataLoader
(
BaseDataLoader
):
...
...
@@ -138,8 +139,8 @@ class FallDataLoader(BaseDataLoader):
df
=
file_reader
.
read_csv
(
infile
,
converters
=
self
.
converters
)
X
=
df
.
drop
([
'Fall'
],
axis
=
1
)
y
=
df
[
'Fall'
]
self
.
X
=
X
self
.
y
=
y
self
.
X
=
pd
.
DataFrame
(
X
)
self
.
y
=
np
.
array
(
y
)
return
self
class
RiskDataLoader
(
BaseDataLoader
):
...
...
@@ -155,8 +156,8 @@ class RiskDataLoader(BaseDataLoader):
df
=
file_reader
.
read_csv
(
infile
,
converters
=
self
.
converters
)
X
=
df
.
drop
([
'Risk'
],
axis
=
1
)
y
=
df
[
'Risk'
]
self
.
X
=
X
self
.
y
=
y
self
.
X
=
pd
.
DataFrame
(
X
)
self
.
y
=
np
.
array
(
y
)
return
self
class
AlarmDataLoader
(
BaseDataLoader
):
...
...
@@ -173,6 +174,6 @@ class AlarmDataLoader(BaseDataLoader):
X
=
df
.
drop
([
'Status'
,
'Days'
],
axis
=
1
)
y
=
np
.
array
(
list
(
tuple
(
x
)
for
x
in
df
[[
'Status'
,
'Days'
]].
to_numpy
()),
dtype
=
[(
'Status'
,
'bool'
),
(
'Days_to_alarm'
,
'>i4'
)])
self
.
X
=
X
self
.
y
=
y
self
.
X
=
pd
.
DataFrame
(
X
)
self
.
y
=
np
.
array
(
y
)
return
self
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment