Convert Excel files to CSV - Uncategorised files (#1563)

src/scripts/schistosomiasis/schisto_analysis.py

@@ -13,6 +13,7 @@
 from tlo import Date, Simulation, logging
 from tlo.analysis.utils import parse_log_file
 from tlo.methods import contraception, demography, healthburden, healthsystem, schisto
+from tlo.util import read_csv_files
 
 
 def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True):
@@ -136,7 +137,7 @@ def save_general_outputs_and_params():
     # loger_DALY_All.to_csv(savepath_daly, index=False)
 
     # parameters spreadsheet
-    parameters_used = pd.read_excel(Path("./resources/ResourceFile_Schisto.xlsx"), sheet_name=None)
+    parameters_used = read_csv_files(Path("./resources/ResourceFile_Schisto"), files=None)
     writer = pd.ExcelWriter(savepath_params)
     for sheet_name in parameters_used.keys():
         parameters_used[sheet_name].to_excel(writer, sheet_name=sheet_name)
@@ -245,8 +246,8 @@ def get_values_per_district(infection):
 
 
 def get_expected_prevalence(infection):
-    expected_district_prevalence = pd.read_excel(Path("./resources") / 'ResourceFile_Schisto.xlsx',
-                                                 sheet_name='District_Params_' + infection.lower())
+    expected_district_prevalence = read_csv_files(Path("./resources") / 'ResourceFile_Schisto',
+                                                 files='District_Params_' + infection.lower())
     expected_district_prevalence = \
         expected_district_prevalence[expected_district_prevalence['District'].isin(['Blantyre',
                                                                                     'Chiradzulu', 'Mulanje', 'Nsanje',

src/tlo/analysis/utils.py

@@ -23,7 +23,11 @@
 
 from tlo import Date, Simulation, logging, util
 from tlo.logging.reader import LogData
-from tlo.util import create_age_range_lookup
+from tlo.util import (
+    create_age_range_lookup,
+    parse_csv_values_for_columns_with_mixed_datatypes,
+    read_csv_files,
+)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -1391,19 +1395,19 @@ def construct_multiindex_if_implied(df):
                 squeeze_single_col_df_to_series(
                     drop_extra_columns(
                         construct_multiindex_if_implied(
-                            pd.read_excel(workbook, sheet_name=sheet_name))))
+                            workbook[sheet_name])))
 
         elif isinstance(_value, str) and _value.startswith("["):
             # this looks like its intended to be a list
             return eval(_value)
         else:
             return _value
 
-    workbook = pd.ExcelFile(
-        resourcefilepath / 'ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx')
+    workbook = read_csv_files(
+        resourcefilepath / 'ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking', files=None)
 
     # Load the ResourceFile for the list of parameters that may change
-    mainsheet = pd.read_excel(workbook, 'main').set_index(['Module', 'Parameter'])
+    mainsheet = workbook['main'].set_index(['Module', 'Parameter'])
 
     # Select which columns for parameter changes to extract
     cols = []
@@ -1416,6 +1420,7 @@ def construct_multiindex_if_implied(df):
     # Collect parameters that will be changed (collecting the first encountered non-NAN value)
     params_to_change = mainsheet[cols].dropna(axis=0, how='all')\
                                       .apply(lambda row: [v for v in row if not pd.isnull(v)][0], axis=1)
+    params_to_change = params_to_change.apply(parse_csv_values_for_columns_with_mixed_datatypes)
 
     # Convert to dictionary
     params = defaultdict(lambda: defaultdict(dict))

src/tlo/methods/enhanced_lifestyle.py

@@ -13,7 +13,7 @@
 from tlo.events import PopulationScopeEventMixin, RegularEvent
 from tlo.lm import LinearModel, LinearModelType, Predictor
 from tlo.logging.helpers import grouped_counts_with_all_combinations
-from tlo.util import get_person_id_to_inherit_from
+from tlo.util import get_person_id_to_inherit_from, read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -343,9 +343,9 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
         p = self.parameters
-        dataframes = pd.read_excel(
-            Path(self.resourcefilepath) / 'ResourceFile_Lifestyle_Enhanced.xlsx',
-            sheet_name=["parameter_values", "urban_rural_by_district"],
+        dataframes = read_csv_files(
+            Path(self.resourcefilepath) / 'ResourceFile_Lifestyle_Enhanced',
+            files=["parameter_values", "urban_rural_by_district"],
         )
         self.load_parameters_from_dataframe(dataframes["parameter_values"])
         p['init_p_urban'] = (

src/tlo/methods/stunting.py

@@ -24,6 +24,7 @@
 from tlo.methods import Metadata
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
+from tlo.util import read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -149,8 +150,8 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
         self.load_parameters_from_dataframe(
-            pd.read_excel(
-                Path(self.resourcefilepath) / 'ResourceFile_Stunting.xlsx', sheet_name='Parameter_values')
+            read_csv_files(
+                Path(self.resourcefilepath) / 'ResourceFile_Stunting', files='Parameter_values')
         )
 
     def initialise_population(self, population):