Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
USMortality committed Apr 5, 2023
1 parent 1e5a374 commit 7805fd3
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 19 deletions.
13 changes: 7 additions & 6 deletions archive.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
[[ $OSTYPE == 'darwin'* ]] && alias date=gdate

function import_csv() {
cd tools
Expand Down Expand Up @@ -30,26 +31,26 @@ function archive() {
mysql -h 127.0.0.1 -u root -e "SET GLOBAL local_infile=1;"

if [ "$1" = "project" ]; then
start=$(gdate -d "(date) - 10 weeks" +%F)
start=$(date -d "(date) - 10 weeks" +%F)
else
start=$(gdate -d "(date) - 3 weeks" +%F)
start=$(date -d "(date) - 3 weeks" +%F)
fi
end=$(gdate -d "(date) - 3 weeks" +%F)
end=$(date -d "(date) - 3 weeks" +%F)

# Population
mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS population;"
import_csv population.csv population
import_csv population_std.csv population

# Single week
# week=$(gdate -d $end +%Y)"_"$(gdate -d $end +%U)
# week=$(date -d $end +%Y)"_"$(date -d $end +%U)
# archive $week

# Process death files for last n weeks
mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS archive;"
while ! [[ $start > $end ]]; do
start=$(gdate -d "$start + 1 week" +%F)
week=$(gdate -d $start +%Y)"_"$(gdate -d $start +%U)
start=$(date -d "$start + 1 week" +%F)
week=$(date -d $start +%Y)"_"$(date -d $start +%U)
archive "${week}"
done

Expand Down
13 changes: 7 additions & 6 deletions de.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
[[ $OSTYPE == 'darwin'* ]] && alias date=gdate

function import_csv() {
cd tools
Expand All @@ -7,8 +8,8 @@ function import_csv() {
}

# Download latest data
start=$(gdate -d "(date) - 2 weeks" +%F)
week=$(gdate -d $start +%Y)"_"$(gdate -d $start +%U)
start=$(date -d "(date) - 2 weeks" +%F)
week=$(date -d $start +%Y)"_"$(date -d $start +%U)
wget https://s3.mortality.watch/data/mortality/deu/deaths.csv \
-O data/de/Tote_${week}.csv

Expand All @@ -22,12 +23,12 @@ rm data/population.tsv data/einwohner.csv

ln -sf "de/esp2013.csv" "data/population_std.csv"

start=$(gdate -d "(date) - 10 weeks" +%F)
end=$(gdate -d "(date) - 3 weeks" +%F)
start=$(date -d "(date) - 10 weeks" +%F)
end=$(date -d "(date) - 3 weeks" +%F)

while ! [[ $start > $end ]]; do
start=$(gdate -d "$start + 1 week" +%F)
week=$(gdate -d $start +%Y)"_"$(gdate -d $start +%U)
start=$(date -d "$start + 1 week" +%F)
week=$(date -d $start +%Y)"_"$(date -d $start +%U)

ln -sf "de/Tote_${week}.csv" "data/deaths.csv"
import_csv deaths.csv deaths
Expand Down
4 changes: 2 additions & 2 deletions tools/import_csv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ cd $(dirname $FILE_PATH)
ln -s "$FILE" "$TABLE_NAME.csv"

# Import data.
mysqlimport --local -h 127.0.0.1 -u root --columns=$COLUMNS --fields-terminated-by="," --fields-optionally-enclosed-by='"' --ignore-lines=1 $DB "$TABLE_NAME.csv"
mysqlimport --local -h 127.0.0.1 -u root --fields-terminated-by="," --fields-optionally-enclosed-by='"' --ignore-lines=1 $DB "$TABLE_NAME.csv"

unlink "$TABLE_NAME.csv"
cd ~-
cd ~-
11 changes: 6 additions & 5 deletions us.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
[[ $OSTYPE == 'darwin'* ]] && alias date=gdate

function import_csv() {
cd tools
Expand All @@ -7,7 +8,7 @@ function import_csv() {
}

# Update Data
date=$(gdate --date="14 days ago" +"%Y")"_"$(gdate --date="14 days ago" +"%U")
date=$(date --date="14 days ago" +"%Y")"_"$(date --date="14 days ago" +"%U")
wget "https://data.cdc.gov/api/views/y5bj-9g5w/rows.csv?accessType=DOWNLOAD" \
-O "data/us/Weekly_counts_of_deaths_by_jurisdiction_and_age_group_${date}.csv"

Expand All @@ -22,16 +23,16 @@ mysql -h 127.0.0.1 -u root -e "SET GLOBAL sql_mode = '';"
ln -sf "us/population20152021.csv" "data/population.csv"
ln -sf "us/std_population2000.csv" "data/population_std.csv"

start=$(gdate -d "(date) - 10 weeks" +%F)
end=$(gdate -d "(date) - 3 weeks" +%F)
start=$(date -d "(date) - 10 weeks" +%F)
end=$(date -d "(date) - 3 weeks" +%F)

# Import Covid Deaths
ln -sf "us/Weekly_Counts_of_Deaths_by_State_and_Select_Causes_2014-2019.csv" "data/covid_deaths_2014-2019.csv"
import_csv "covid_deaths_2014-2019.csv" deaths

while ! [[ $start > $end ]]; do
start=$(gdate -d "$start + 1 week" +%F)
week=$(gdate -d $start +%Y)"_"$(gdate -d $start +%U)
start=$(date -d "$start + 1 week" +%F)
week=$(date -d $start +%Y)"_"$(date -d $start +%U)
echo "Week $week"

ln -sf "us/Weekly_counts_of_deaths_by_jurisdiction_and_age_group_${week}.csv" "data/deaths.csv"
Expand Down

0 comments on commit 7805fd3

Please sign in to comment.