Commit 9b1da254 authored by Lars Dalby's avatar Lars Dalby
Browse files

Merge branch 'change-handling-of-data-collection' into 'master'

Change handling of data collection

See merge request LDalby/almass!36
parents e59ba107 248cac58
......@@ -46,4 +46,6 @@ Temporary Items
**/Grendel/.Rproj.user
**/Grendel/.Rhistory
**/model_testing_report/GooseModelTesting.html
\ No newline at end of file
**/model_testing_report/GooseModelTesting.html
**/Grendel/scenarios/numbers_report.html
**/Grendel/param_fit_report/param-fit-goose.html
\ No newline at end of file
......@@ -5,7 +5,7 @@
#SBATCH --mem=240G
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=4
#SBATCH --time=48:00:00
#SBATCH --time=168:00:00
#SBATCH --exclusive
#SBATCH --mem-per-cpu=60G
#SBATCH --mail-type=ALL
......
......@@ -5,7 +5,7 @@
#SBATCH --mem=240G
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=8
#SBATCH --time=48:00:00
#SBATCH --time=168:00:00
#SBATCH --exclusive
#SBATCH --mem-per-cpu=40G
#SBATCH --mail-type=ALL
......
......@@ -23,23 +23,40 @@ plan(multiprocess) # to run in parallel future_map in parallel.
suffix <- stringr::str_pad(as.character(1:npar), width = 2, pad = "0")
path(path_to_dirs, glue::glue("WD{suffix}"), "Results", file_name) %>%
future_map(data.table::fread) %>%
data.table::rbindlist() %>%
data.table::setnames(c("season", "day", "pinkfoot", "pinkfoot_dist",
tibble::tibble(file_name = .,
wd = glue::glue("WD{suffix}")) %>%
dplyr::mutate(file_contents = future_map(file_name, ~data.table::fread(file = .))) %>%
tidyr::unnest() %>%
dplyr::select(-file_name) %>%
purrr::set_names(nm = c("wd", "season", "day", "pinkfoot", "pinkfoot_dist",
"barnacle", "barnacle_dist", "greylag", "greylag_dist",
"grain", "maize", "polyref", "param", "value")) -> all_res
grain_file_name <- "scenario-grain-distributions.txt"
path(path_to_dirs, glue::glue("WD{suffix}"), "Results", grain_file_name) %>%
future_map(data.table::fread) %>%
data.table::rbindlist() %>%
data.table::setnames(c("season", "grain_dist", "param", "value")) -> all_grain
tibble::tibble(file_name = .,
wd = glue::glue("WD{suffix}")) %>%
dplyr::mutate(file_contents = future_map(file_name, ~data.table::fread(file = .))) %>%
tidyr::unnest() %>%
dplyr::select(-file_name) %>%
purrr::set_names(nm = c("wd", "season", "grain_dist", "param", "value")) -> all_grain
rotation_file_name <- "scenario-rotations.txt"
path(path_to_dirs, glue::glue("WD{suffix}"), rotation_file_name) %>%
tibble::tibble(file_name = .) %>%
dplyr::mutate(file_contents = future_map(file_name, ~data.table::fread(file = .))) %>%
tidyr::unnest() %>%
dplyr::select(-file_name) %>%
purrr::set_names(c("wd", "rotation")) -> all_rotation
numbers_file_name <- "scenario-goose-numbers.txt"
path(path_to_dirs, glue::glue("WD{suffix}"), "Results", numbers_file_name) %>%
future_map(data.table::fread) %>%
data.table::rbindlist() %>%
data.table::setnames(c("season", "day", "pf_families", "pf_non_breeders", "bn_families",
tibble::tibble(file_name = .,
wd = glue::glue("WD{suffix}")) %>%
dplyr::mutate(file_contents = future_map(file_name, ~data.table::fread(file = .))) %>%
tidyr::unnest() %>%
dplyr::select(-file_name) %>%
purrr::set_names(nm = c("wd", "season", "day", "pf_families", "pf_non_breeders", "bn_families",
"bn_non_breeders", "gl_families", "gl_non_breeders", "snow_depth",
"param", "value")) -> all_numbers
......@@ -51,6 +68,9 @@ fst::write_fst(all_res, path = path(dest_dir, res_file_name))
grain_file_name <- glue::glue("{args}_maize-availability-grain-distributions_{Sys.Date()}.fst")
fst::write_fst(all_grain, path = path(dest_dir, grain_file_name))
rotation_file_name <- glue::glue("{args}_maize-availability-rotations_{Sys.Date()}.fst")
fst::write_fst(all_rotation, path = path(dest_dir, rotation_file_name))
numbers_file_name <- glue::glue("{args}_maize-availability-goose-numbers_{Sys.Date()}.fst")
fst::write_fst(all_numbers, path = path(dest_dir, numbers_file_name))
......
......@@ -8,10 +8,15 @@ cp /home/ldalby/ALMaSS/almass/Grendel/FileDist.r /home/ldalby/workspace/Goose/Pa
cp /home/ldalby/ALMaSS/almass/Grendel/file-dist-fixed-ranges.R /home/ldalby/workspace/Goose/ParamFitting
cp /home/ldalby/ALMaSS/almass/Grendel/CollectResults.r /home/ldalby/workspace/Goose/ParamFitting
cp /home/ldalby/ALMaSS/almass/Grendel/collect-results.R /home/ldalby/workspace/Goose/ParamFitting
cp /home/ldalby/ALMaSS/almass/Grendel/rotations/UserDefinedFarm19.rot /home/ldalby/workspace/Goose/RunDirectory
# Scenarios
# 01
cp /home/ldalby/ALMaSS/almass/Grendel/scenario_01_goose-numbers_file-dist.R /home/ldalby/workspace/Goose/Scenarios
cp /home/ldalby/ALMaSS/almass/Grendel/collect-results-scenario_01.R /home/ldalby/workspace/Goose/Scenarios
cp /home/ldalby/ALMaSS/almass/Grendel/scenario_01_goose-numbers.R /home/ldalby/workspace/Goose/RunDirectory
# 02
cp /home/ldalby/ALMaSS/almass/Grendel/scenario_02_maize-availability_file-dist.R /home/ldalby/workspace/Goose/Scenarios
cp /home/ldalby/ALMaSS/almass/Grendel/collect-results-scenario_02.R /home/ldalby/workspace/Goose/Scenarios
cp /home/ldalby/ALMaSS/almass/Grendel/goosebatchr.r /home/ldalby/workspace/Goose/RunDirectory
cp /home/ldalby/ALMaSS/almass/Grendel/PreRunSetup.r /home/ldalby/workspace/Goose/RunDirectory
......
......@@ -134,7 +134,7 @@ seq(1.0, 1.1, length.out = nsteps) %>%
# Edit the bat, ini and cfg files to match the parameters set above:
for (i in seq_along(dirs)) {
wd = path(pathtodirs, dirs[i])
EditBat(wd)
edit_bat(wd, "_01_BatchLoop.sh")
EditIni(WorkDir = wd, Model = "goose", NYear = years + 1)
EditConfig(file = path(wd, "TIALMaSSConfig.cfg"), config = "GOOSE_MODELEXITDAY", value = 31 + years*365)
EditConfig(file = path(wd, "TIALMaSSConfig.cfg"), config = "MAP_WEATHER_FILE", value = "Vejlerne2011-2014.pre")
......
......@@ -35,7 +35,6 @@ error_path_remote <- "/home/ldalby/workspace/Goose/Scenarios/Errors/"
error_files <- path(error_path_remote, glue("{jobs}*"))
walk2(.x = error_files, .y = error_path, scp_download, session = session)
summarized_remote <- path(res_path_remote, glue("scenario_results_{jobs}*"))
summarized_remote <- path(res_path_remote, glue("*{jobs}*"))
walk2(.x = summarized_remote, .y = path(current, jobs), scp_download, session = session)
......
......@@ -14,7 +14,7 @@ output:
editor_options:
chunk_output_type: console
params:
date_stamp: "2018-08-22"
date_stamp: "2019-01-23"
---
```{r setup, include=FALSE}
......
......@@ -10,14 +10,14 @@ SeedGrass1
SpringBarley
WinterWheat
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleyCloverGrass
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
WinterWheat
......@@ -29,29 +29,29 @@ SpringBarleyCloverGrass
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
WinterBarley
WinterRape
MaizeSilage
SpringBarley
SpringBarleySpr
Oats
WinterWheat
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleyCloverGrass
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleySilage
......@@ -63,11 +63,11 @@ SpringBarleyCloverGrass
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleySilage
......@@ -76,29 +76,29 @@ CloverGrassGrazed1
CloverGrassGrazed2
WinterWheat
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleySilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterBarley
WinterRape
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterWheat
WinterRye
SpringBarleySilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
CloverGrassGrazed2
......@@ -10,14 +10,14 @@ SeedGrass1
SpringBarley
WinterWheat
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
MaizeSilage
CloverGrassGrazed2
MaizeSilage
CloverGrassGrazed1
MaizeSilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
WinterWheat
......@@ -29,22 +29,22 @@ SpringBarleyCloverGrass
CloverGrassGrazed1
MaizeSilage
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
WinterBarley
WinterRape
MaizeSilage
SpringBarley
SpringBarleySpr
Oats
WinterWheat
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleyCloverGrass
......@@ -54,7 +54,7 @@ MaizeSilage
MaizeSilage
CloverGrassGrazed1
MaizeSilage
SpringBarleySilage
SpringBarleySpr
SpringBarleyCloverGrass
CloverGrassGrazed1
CloverGrassGrazed2
......@@ -67,7 +67,7 @@ MaizeSilage
CloverGrassGrazed1
MaizeSilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleySilage
......@@ -80,25 +80,25 @@ MaizeSilage
CloverGrassGrazed1
MaizeSilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
SpringBarleySilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
MaizeSilage
CloverGrassGrazed2
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterBarley
WinterRape
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterWheat
WinterRye
SpringBarleySilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
MaizeSilage
CloverGrassGrazed2
......@@ -10,7 +10,7 @@ SeedGrass1
SpringBarley
WinterWheat
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
MaizeSilage
CloverGrassGrazed2
MaizeSilage
......@@ -20,46 +20,46 @@ MaizeSilage
MaizeSilage
CloverGrassGrazed1
MaizeSilage
SpringBarleyCloverGrass
WinterWheat
WinterRye
SpringBarleyCloverGrass
MaizeSilage
CloverGrassGrazed2
SpringBarleyCloverGrass
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
WinterBarley
WinterRape
MaizeSilage
SpringBarley
SpringBarleySpr
Oats
WinterWheat
MaizeSilage
MaizeSilage
CloverGrassGrazed1
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
MaizeSilage
CloverGrassGrazed2
MaizeSilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarleySilage
SpringBarleySpr
MaizeSilage
CloverGrassGrazed1
MaizeSilage
WinterWheat
SpringBarleyCloverGrass
WinterWheat
MaizeSilage
CloverGrassGrazed2
MaizeSilage
......@@ -70,7 +70,7 @@ MaizeSilage
MaizeSilage
CloverGrassGrazed1
MaizeSilage
SpringBarleySilage
SpringBarleySpr
SpringBarleyCloverGrass
MaizeSilage
CloverGrassGrazed2
......@@ -83,22 +83,22 @@ MaizeSilage
MaizeSilage
CloverGrassGrazed1
MaizeSilage
SpringBarleySilage
SpringBarleySpr
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
CloverGrassGrazed1
CloverGrassGrazed2
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterBarley
WinterRape
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterWheat
WinterRye
SpringBarleySilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
MaizeSilage
MaizeSilage
CloverGrassGrazed2
......
......@@ -29,18 +29,18 @@ MaizeSilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
MaizeSilage
SpringBarley
SpringBarleySpr
WinterWheat
WinterBarley
WinterRape
MaizeSilage
SpringBarley
SpringBarleySpr
Oats
WinterWheat
MaizeSilage
......@@ -54,10 +54,10 @@ MaizeSilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarleySilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarleySpr
WinterWheat
MaizeSilage
CloverGrassGrazed1
......@@ -70,30 +70,30 @@ MaizeSilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarleySilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarleySpr
WinterWheat
MaizeSilage
MaizeSilage
MaizeSilage
MaizeSilage
MaizeSilage
SpringBarleyCloverGrass
SpringBarleySpr
MaizeSilage
MaizeSilage
SpringBarleySilage
SpringBarleySpr
MaizeSilage
MaizeSilage
CloverGrassGrazed1
MaizeSilage
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterBarley
WinterRape
MaizeSilage
SpringBarleySilage
SpringBarleySpr
WinterWheat
WinterRye
SpringBarleySilage
......
......@@ -22,7 +22,7 @@ basedir <- "/home/ldalby/workspace/Goose/RunDirectory"
pathtodirs <- path('/scratch', args)
# Setup the directories
npar <- 4 # Specifies the number of run directories
years <- 10 # the number of seasons to run (goose sims run over the year boundary)
years <- 100 # the number of seasons to run (goose sims run over the year boundary)
basename <- "WD" # The prefix to the directories
# Make the directories and copy the files:
for (i in 1:npar) {
......
......@@ -22,7 +22,7 @@ basedir <- "/home/ldalby/workspace/Goose/RunDirectory"
pathtodirs <- path("/scratch", args)
# Setup the directories
npar <- 8 # Specifies the number of run directories
years <- 10 # the number of seasons to run (goose sims run over the year boundary)
years <- 100 # the number of seasons to run (goose sims run over the year boundary)
basename <- "WD" # The prefix to the directories
# Make the directories and copy the files:
for (i in 1:npar) {
......@@ -43,84 +43,116 @@ for (i in seq_along(dirs)) {
# Read in the config so we can get the default values:
cfg <- readLines("/home/ldalby/workspace/Goose/RunDirectory/TIALMaSSConfig.cfg")
# Ditribute the different parameters ----
# Distribute the different parameters ----
nsteps <- 10 # the number of intervals to split the parameter in
replace_rotation <- function(new_rot, old_rot, wd_path) {
rot_paths <- "/home/ldalby/ALMaSS_all/almass/Grendel/rotations/"
rot_paths <- "/home/ldalby/ALMaSS/almass/Grendel/rotations/"
focal_rot <- path(rot_paths, new_rot)
file_copy(focal_rot,
new_path = path(wd_path, old_rot),
new_path = path(wd_path, old_rot),
overwrite = TRUE)
}
write_rot_info <- function(dir, rot, wdpath) {
tribble(
~WD, ~maize_rot,
dir, rot) %>%
write_tsv(path = path(wdpath, "scenario-rotations.txt"))
}
old_rot <- "UserDefinedFarm19.rot"
rot_path <- "/home/ldalby/ALMaSS_all/almass/Grendel/rotations"
# Startno scaler - standard maize ----
wdpath <- path(pathtodirs, dirs[1])
current_dir <- dirs[1]
wdpath <- path(pathtodirs, current_dir)
new_rot <- "UserDefinedFarm19.rot"
seq(1, 10, length.out = nsteps) %>%
round() %>%
GenerateParams("GOOSE_STARTNO_SCALER" = ., write = TRUE, path = wdpath)
write_rot_info(current_dir, new_rot, wdpath)
# Startno scaler - maize rotation 191 ----
wdpath <- path(pathtodirs, dirs[1])
current_dir <- dirs[2]
wdpath <- path(pathtodirs, current_dir)
new_rot <- "UserDefinedFarm191.rot"
seq(1, 10, length.out = nsteps) %>%
round() %>%
GenerateParams("GOOSE_STARTNO_SCALER" = ., write = TRUE, path = wdpath)
replace_rotation(new_rot = path(rot_path, "UserDefinedFarm191.rot"),
replace_rotation(new_rot = new_rot,
old_rot = old_rot,
wd_path = wdpath)
write_rot_info(current_dir, new_rot, wdpath)
# Startno scaler - maize rotation 192 ----
wdpath <- path(pathtodirs, dirs[1])
current_dir <- dirs[3]
wdpath <- path(pathtodirs, current_dir)
new_rot <- "UserDefinedFarm192.rot"
seq(1, 10, length.out = nsteps) %>%
round() %>%
GenerateParams("GOOSE_STARTNO_SCALER" = ., write = TRUE, path = wdpath)
replace_rotation(new_rot = path(rot_path, "UserDefinedFarm192.rot"),
replace_rotation(new_rot = new_rot,
old_rot = old_rot,
wd_path = wdpath)
write_rot_info(current_dir, new_rot, wdpath)
# Startno scaler - maize rotation 193 ----
wdpath <- path(pathtodirs, dirs[1])
current_dir <- dirs[4]
wdpath <- path(pathtodirs, current_dir)
new_rot <- "UserDefinedFarm193.rot"
seq(1, 10, length.out = nsteps) %>%
round() %>%
GenerateParams("GOOSE_STARTNO_SCALER" = ., write = TRUE, path = wdpath)
replace_rotation(new_rot = path(rot_path, "UserDefinedFarm193.rot"),
replace_rotation(new_rot = new_rot,
old_rot = old_rot,
wd_path = wdpath)
write_rot_info(current_dir, new_rot, wdpath)
# Startno scaler - standard maize ----
wdpath <- path(pathtodirs, dirs[1])
current_dir <- dirs[5]
wdpath <- path(pathtodirs, current_dir)
new_rot <- "UserDefinedFarm19.rot"
seq(1, 10, length.out = nsteps) %>%
round() %>%
GenerateParams("GOOSE_BN_STARTNO_SCALER" = ., write = TRUE, path = wdpath)
write_rot_info(current_dir, new_rot, wdpath)
# BN Startno scaler - maize rotation 191 ----
wdpath <- path(pathtodirs, dirs[1])
current_dir <- dirs[6]
wdpath <- path(pathtodirs, current_dir)
new_rot <- "UserDefinedFarm191.rot"
seq(1, 10, length.out = nsteps) %>%