etl/config/config.yml

+run_type: constant_abundance_p15_q05
+
+computation:
+    threads: 10
+
+overrides:
+    dust_temperature_grid_type: 'linear'
+    dust_temperature_limits: [10, 35]
+    dust_temperature_step: 2.5
+    gas_density_grid_type: 'log'
+    gas_density_limits: [1e3, 1e7]
+    gas_density_step: 3
+    gas_density_unit: cm^-3
+    lines_to_process: [['87', '86'], ['88', '87'], ['88', '86'], ['257', '256'], ['381', '380']]

etl/config/density_inference_input.yml

+best_bandwidths:
+  257-256: 0.4
+  381-380: 0.2
+  87-86: 0.1
+  88-86: 0.1
+  88-87: 0.1
+integrated_intensities_uncertainties:
+  '256':
+    G08.71-0.41: 0.3772371124870143
+    G10.45-0.02: 0.3438149090944644
+    G10.47+0.03: 4.67062615566759
+    G10.62-0.38: 5.717801509967671
+    G12.81-0.20: 2.088877254036277
+    G13.18+0.06: 1.43381518306444
+    G13.66-0.60: 1.1620189309770297
+    G14.11-0.57: 0.3933488163719442
+    G14.19-0.19: 1.9966333535808136
+    G14.49-0.14: 0.5232957299744114
+    G14.63-0.58: 0.6760059683722098
+    G15.03-0.67: 1.3201864103978675
+    G18.61-0.07: 0.449933406945426
+    G18.73-0.23: 1.1925529389006952
+    G18.89-0.47: 1.2420175138135696
+    G19.88-0.54: 1.1775666735052563
+    G22.37+0.45: 0.8397889239422417
+    G23.21-0.38: 2.366268748274346
+    G28.56-0.24: 0.7853900725084517
+    G28.86+0.07: 0.5
+    G30.85-0.08: 0.5698331975250233
+    G31.41+0.31: 2.794406439049449
+    G34.26+0.15: 5.571485634832507
+    G34.40+0.23: 2.4222527304195376
+    G34.41+0.23: 2.455950237377704
+    G34.82+0.35: 0.1947198188044887
+    G35.20-0.74: 1.9116488388286448
+    G37.55+0.20: 0.7800057066048043
+    G49.49-0.39: 13.79415939155848
+    G53.14+0.07: 0.9832922403050368
+    G59.78+0.07: 0.5
+  '257':
+    G08.71-0.41: 0.4136351969963581
+    G10.45-0.02: 0.3567836016634692
+    G10.47+0.03: 4.737191542189759
+    G10.62-0.38: 5.828217319934533
+    G12.81-0.20: 2.159742388795141
+    G13.18+0.06: 1.528153771626887
+    G13.66-0.60: 1.2364636458899043
+    G14.11-0.57: 0.4315807681683414
+    G14.19-0.19: 2.1114871424317903
+    G14.49-0.14: 0.5631062582040194
+    G14.63-0.58: 0.7362604233371383
+    G15.03-0.67: 1.3883181061379544
+    G18.61-0.07: 0.4918960784434188
+    G18.73-0.23: 1.2489705007902534
+    G18.89-0.47: 1.30997858860221
+    G19.88-0.54: 1.2556873263733366
+    G22.37+0.45: 0.9012219713131504
+    G23.21-0.38: 2.4463347346958884
+    G28.56-0.24: 0.8256372552866202
+    G28.86+0.07: 0.5
+    G30.85-0.08: 0.5939589107713851
+    G31.41+0.31: 2.830674270184918
+    G34.26+0.15: 5.689969110997285
+    G34.40+0.23: 2.5501966207773785
+    G34.41+0.23: 2.586820094338446
+    G34.82+0.35: 0.2153653834587912
+    G35.20-0.74: 2.009289929523078
+    G37.55+0.20: 0.8302500355158099
+    G49.49-0.39: 13.996345324993044
+    G53.14+0.07: 1.037736129080711
+    G59.78+0.07: 0.5
+  '380':
+    G08.71-0.41: 0.1719132347425028
+    G10.45-0.02: 0.3350748768496078
+    G10.47+0.03: 2.5238526974413102
+    G10.62-0.38: 1.9730421303059005
+    G12.81-0.20: 0.4938645969760634
+    G13.18+0.06: 0.3130453732259515
+    G13.66-0.60: 0.3896020198637059
+    G14.11-0.57: 0.1604869867792384
+    G14.19-0.19: 0.3718943617922549
+    G14.49-0.14: 0.3980987437341203
+    G14.63-0.58: 0.1942770443673398
+    G15.03-0.67: 0.4226996779770496
+    G18.61-0.07: 0.3104707669438686
+    G18.73-0.23: 0.3983825447868095
+    G18.89-0.47: 0.3283273384950035
+    G19.88-0.54: 0.297825582498799
+    G22.37+0.45: 0.2435883035421827
+    G23.21-0.38: 0.4720878606853434
+    G28.56-0.24: 0.3689771528703325
+    G28.86+0.07: 0.2133053661117121
+    G30.85-0.08: 0.4002321844656681
+    G31.41+0.31: 1.7677726204007878
+    G34.26+0.15: 2.2053525649662755
+    G34.40+0.23: 0.3157737464671486
+    G34.41+0.23: 0.552143182260122
+    G34.82+0.35: 0.3762696495121889
+    G35.20-0.74: 0.3277361642527309
+    G37.55+0.20: 0.3505786313165484
+    G49.49-0.39: 6.458962659096731
+    G53.14+0.07: 0.2659096119306583
+    G59.78+0.07: 0.1958691834258987
+  '381':
+    G08.71-0.41: 0.1912145881150111
+    G10.45-0.02: 0.348675803268881
+    G10.47+0.03: 2.527338233134208
+    G10.62-0.38: 2.000038955895338
+    G12.81-0.20: 0.5078276831867471
+    G13.18+0.06: 0.3327236010567068
+    G13.66-0.60: 0.4023699128083662
+    G14.11-0.57: 0.1778607797165793
+    G14.19-0.19: 0.3901217659238757
+    G14.49-0.14: 0.410805519891472
+    G14.63-0.58: 0.2096518477508132
+    G15.03-0.67: 0.4404189229260346
+    G18.61-0.07: 0.3250267274896626
+    G18.73-0.23: 0.4111428033705079
+    G18.89-0.47: 0.3562705793927619
+    G19.88-0.54: 0.3054447929243074
+    G22.37+0.45: 0.2565267403612208
+    G23.21-0.38: 0.47971744770208
+    G28.56-0.24: 0.3858032381571517
+    G28.86+0.07: 0.2244369344927569
+    G30.85-0.08: 0.4287274799425559
+    G31.41+0.31: 1.7849749705573348
+    G34.26+0.15: 2.2213861922198466
+    G34.40+0.23: 0.3283727163284978
+    G34.41+0.23: 0.5631980961359404
+    G34.82+0.35: 0.3873761358129582
+    G35.20-0.74: 0.3362751271550414
+    G37.55+0.20: 0.3531542428043672
+    G49.49-0.39: 6.498419983328799
+    G53.14+0.07: 0.278825242987693
+    G59.78+0.07: 0.2103971057794001
+  '86':
+    G08.71-0.41: 0.1141338092612959
+    G10.45-0.02: 0.1513227343419588
+    G10.47+0.03: 0.2436313155314365
+    G10.62-0.38: 0.2543853719744313
+    G12.81-0.20: 0.2120144583210657
+    G13.18+0.06: 0.1444849558699236
+    G13.66-0.60: 0.1880743486996585
+    G14.11-0.57: 0.0991920101690233
+    G14.19-0.19: 0.2591102799195194
+    G14.49-0.14: 0.1932137455379076
+    G14.63-0.58: 0.0920867956240891
+    G15.03-0.67: 0.1519740165468335
+    G18.61-0.07: 0.1336149687864388
+    G18.73-0.23: 0.1544601974519147
+    G18.89-0.47: 0.284814022368405
+    G19.88-0.54: 0.1238378758928978
+    G22.37+0.45: 0.1935045511182322
+    G23.21-0.38: 0.1601721389776916
+    G28.56-0.24: 0.1699145015331222
+    G28.86+0.07: 0.1037452815160559
+    G30.85-0.08: 0.1481775324641313
+    G31.41+0.31: 0.2194502880473983
+    G34.26+0.15: 0.1627530591328137
+    G34.40+0.23: 0.1351065672329636
+    G34.41+0.23: 0.2465062371409508
+    G34.82+0.35: 0.0865664840020827
+    G35.20-0.74: 0.1340658506805993
+    G37.55+0.20: 0.1253184895008262
+    G49.49-0.39: 0.2791842617996957
+    G53.14+0.07: 0.0768660912353171
+    G59.78+0.07: 0.0574168367111523
+  '87':
+    G08.71-0.41: 0.111937365388756
+    G10.45-0.02: 0.1491063536449446
+    G10.47+0.03: 0.2419807697773685
+    G10.62-0.38: 0.2507814699869695
+    G12.81-0.20: 0.2090805176901697
+    G13.18+0.06: 0.1418278810377906
+    G13.66-0.60: 0.1841371839860628
+    G14.11-0.57: 0.097470593166154
+    G14.19-0.19: 0.2519127822081297
+    G14.49-0.14: 0.188762628907307
+    G14.63-0.58: 0.0892839165361122
+    G15.03-0.67: 0.1499962128329128
+    G18.61-0.07: 0.1315931235851593
+    G18.73-0.23: 0.1517892338894
+    G18.89-0.47: 0.2784484436939946
+    G19.88-0.54: 0.121808079112921
+    G22.37+0.45: 0.1892876467707792
+    G23.21-0.38: 0.1576533484208159
+    G28.56-0.24: 0.1640239796382693
+    G28.86+0.07: 0.1023581134022235
+    G30.85-0.08: 0.145424188960802
+    G31.41+0.31: 0.2170064162555401
+    G34.26+0.15: 0.16145840413561
+    G34.40+0.23: 0.1324170108062351
+    G34.41+0.23: 0.2427485184738539
+    G34.82+0.35: 0.0851539809088609
+    G35.20-0.74: 0.1319126641393791
+    G37.55+0.20: 0.122848111292009
+    G49.49-0.39: 0.2773996831748604
+    G53.14+0.07: 0.075643530717601
+    G59.78+0.07: 0.0555749088316803
+  '88':
+    G08.71-0.41: 0.1118095857454675
+    G10.45-0.02: 0.1483259532833787
+    G10.47+0.03: 0.2409668716291754
+    G10.62-0.38: 0.2495329007627918
+    G12.81-0.20: 0.2084495919578603
+    G13.18+0.06: 0.1414209323698791
+    G13.66-0.60: 0.1831553534762108
+    G14.11-0.57: 0.0972920392752167
+    G14.19-0.19: 0.2507189304794556
+    G14.49-0.14: 0.1882827587313301
+    G14.63-0.58: 0.0890047760093749
+    G15.03-0.67: 0.1495172793656858
+    G18.61-0.07: 0.1313898333367211
+    G18.73-0.23: 0.1513505926924855
+    G18.89-0.47: 0.2779301291723239
+    G19.88-0.54: 0.121507432969849
+    G22.37+0.45: 0.1889958485649907
+    G23.21-0.38: 0.1568877828888521
+    G28.56-0.24: 0.1635890088005906
+    G28.86+0.07: 0.1020862031904043
+    G30.85-0.08: 0.1451689199961562
+    G31.41+0.31: 0.2158655413134478
+    G34.26+0.15: 0.1607951465382442
+    G34.40+0.23: 0.1320891607520888
+    G34.41+0.23: 0.2421195456012388
+    G34.82+0.35: 0.0850472508528964
+    G35.20-0.74: 0.1316317053747658
+    G37.55+0.20: 0.1222906026734552
+    G49.49-0.39: 0.2758738141000653
+    G53.14+0.07: 0.0755106793427082
+    G59.78+0.07: 0.0553352170005506
+measured_integrated_intensities:
+  '256':
+    G08.71-0.41: 0.2225977131194544
+    G10.45-0.02: 0.6483658492610417
+    G10.47+0.03: 12.781079959863508
+    G10.62-0.38: 14.026812217900536
+    G12.81-0.20: 4.594177385112016
+    G13.18+0.06: 2.2560621827614065
+    G13.66-0.60: 2.2460391103183417
+    G14.11-0.57: 0.4209070251778891
+    G14.19-0.19: 3.080405790891515
+    G14.49-0.14: 0.7458559310555399
+    G14.63-0.58: 1.2815176351639286
+    G15.03-0.67: 2.5627713663667446
+    G18.61-0.07: 0.6336927890231682
+    G18.73-0.23: 1.7197328016314029
+    G18.89-0.47: 2.044797825251944
+    G19.88-0.54: 2.689820011325686
+    G22.37+0.45: 1.0252721670025875
+    G23.21-0.38: 5.128448315345341
+    G28.56-0.24: 0.3548360822544717
+    G28.86+0.07: 0.1
+    G30.85-0.08: 0.5697324971999391
+    G31.41+0.31: 8.11815427336179
+    G34.26+0.15: 14.800657803071704
+    G34.40+0.23: 3.986394686990907
+    G34.41+0.23: 5.298745541451453
+    G34.82+0.35: 0.2244740966733627
+    G35.20-0.74: 3.557121906245741
+    G37.55+0.20: 1.45314493723405
+    G49.49-0.39: 36.47299209277803
+    G53.14+0.07: 1.769951086391564
+    G59.78+0.07: 0.1
+  '257':
+    G08.71-0.41: 1.6896248544507602
+    G10.45-0.02: 1.6172560812119725
+    G10.47+0.03: 17.90790753460651
+    G10.62-0.38: 23.51945555648909
+    G12.81-0.20: 9.576422834532323
+    G13.18+0.06: 6.614500597410293
+    G13.66-0.60: 5.099529057565239
+    G14.11-0.57: 2.328801003580471
+    G14.19-0.19: 8.648952235029466
+    G14.49-0.14: 2.4117084959115784
+    G14.63-0.58: 4.22783491577442
+    G15.03-0.67: 6.482321394374812
+    G18.61-0.07: 2.119319392382522
+    G18.73-0.23: 4.953380016604151
+    G18.89-0.47: 5.763854872422209
+    G19.88-0.54: 6.538781173715317
+    G22.37+0.45: 3.680142077328448
+    G23.21-0.38: 9.457619850247136
+    G28.56-0.24: 2.68014308698633
+    G28.86+0.07: 0.1
+    G30.85-0.08: 2.087101701007916
+    G31.41+0.31: 10.657245054323353
+    G34.26+0.15: 23.76415857208871
+    G34.40+0.23: 10.868969226850243
+    G34.41+0.23: 11.460979034617344
+    G34.82+0.35: 1.0392490902135514
+    G35.20-0.74: 9.236714367808798
+    G37.55+0.20: 3.44395156070276
+    G49.49-0.39: 53.88508148720575
+    G53.14+0.07: 4.882683930286012
+    G59.78+0.07: 0.1
+  '380':
+    G08.71-0.41: 0.1764976739921008
+    G10.45-0.02: 0.3956084197099886
+    G10.47+0.03: 17.205686694830824
+    G10.62-0.38: 16.829857896249838
+    G12.81-0.20: 3.77800331658647
+    G13.18+0.06: 1.7426684781184258
+    G13.66-0.60: 2.050893367850886
+    G14.11-0.57: 0.4213659117392063
+    G14.19-0.19: 3.0914972380702115
+    G14.49-0.14: 0.5203058400694469
+    G14.63-0.58: 0.7159602280790358
+    G15.03-0.67: 2.1151044891765216
+    G18.61-0.07: 0.6989085703534818
+    G18.73-0.23: 2.341561030185213
+    G18.89-0.47: 1.1824614382740015
+    G19.88-0.54: 5.43307241666044
+    G22.37+0.45: 0.8920116613769251
+    G23.21-0.38: 6.836653617571014
+    G28.56-0.24: 0.0440146014079702
+    G28.86+0.07: 1.4210884372776502
+    G30.85-0.08: 0.5707347171006769
+    G31.41+0.31: 12.47776135190902
+    G34.26+0.15: 22.863445215418928
+    G34.40+0.23: 3.531757873095086
+    G34.41+0.23: 7.573594043835107
+    G34.82+0.35: 0.5068421046687672
+    G35.20-0.74: 3.735886344534172
+    G37.55+0.20: 2.1481844033314563
+    G49.49-0.39: 59.72041757348511
+    G53.14+0.07: 2.1621252984074437
+    G59.78+0.07: 0.9953421793454204
+  '381':
+    G08.71-0.41: 0.6722454928574747
+    G10.45-0.02: 0.9251788351499816
+    G10.47+0.03: 18.155676944590947
+    G10.62-0.38: 26.31767222497715
+    G12.81-0.20: 6.359416798755954
+    G13.18+0.06: 5.105986039174396
+    G13.66-0.60: 3.5815529807897217
+    G14.11-0.57: 1.4017924550811132
+    G14.19-0.19: 7.297072065038203
+    G14.49-0.14: 1.440371736841573
+    G14.63-0.58: 3.008997339860133
+    G15.03-0.67: 5.842826943986542
+    G18.61-0.07: 1.8679581479443947
+    G18.73-0.23: 4.779917207406943
+    G18.89-0.47: 3.8325153184359935
+    G19.88-0.54: 10.285222566660105
+    G22.37+0.45: 2.6850343064569504
+    G23.21-0.38: 9.739053502995969
+    G28.56-0.24: 0.9680884829289998
+    G28.86+0.07: 2.7304343388726755
+    G30.85-0.08: 1.7951582288444188
+    G31.41+0.31: 15.666989012027273
+    G34.26+0.15: 27.670510995076157
+    G34.40+0.23: 8.838480648374599
+    G34.41+0.23: 12.543875121955834
+    G34.82+0.35: 0.912504455201019
+    G35.20-0.74: 8.213603364738846
+    G37.55+0.20: 2.427780024280013
+    G49.49-0.39: 71.80734717626261
+    G53.14+0.07: 4.9231567416211615
+    G59.78+0.07: 2.24196075698127
+  '86':
+    G08.71-0.41: 5.770415738358244
+    G10.45-0.02: 3.585944537913708
+    G10.47+0.03: 19.61767281185442
+    G10.62-0.38: 17.24610289004276
+    G12.81-0.20: 8.431310860817968
+    G13.18+0.06: 10.867003106610996
+    G13.66-0.60: 8.534961548014586
+    G14.11-0.57: 3.616973808521042
+    G14.19-0.19: 15.749893351960294
+    G14.49-0.14: 6.365582995343145
+    G14.63-0.58: 5.623662981046849
+    G15.03-0.67: 4.060482703151656
+    G18.61-0.07: 4.924693385096532
+    G18.73-0.23: 10.095062635647638
+    G18.89-0.47: 13.902280291349596
+    G19.88-0.54: 7.349704071521969
+    G22.37+0.45: 8.934333184369493
+    G23.21-0.38: 10.589381466381129
+    G28.56-0.24: 7.718943160772005
+    G28.86+0.07: 3.744479978305277
+    G30.85-0.08: 3.600081120187853
+    G31.41+0.31: 10.334408688143643
+    G34.26+0.15: 17.148782015804713
+    G34.40+0.23: 14.72159545617476
+    G34.41+0.23: 17.151716147853218
+    G34.82+0.35: 2.3557453894152305
+    G35.20-0.74: 11.441136922618975
+    G37.55+0.20: 5.218930784342653
+    G49.49-0.39: 40.047671620058914
+    G53.14+0.07: 3.998167013285548
+    G59.78+0.07: 2.88330892529155
+  '87':
+    G08.71-0.41: 1.3070165152749178
+    G10.45-0.02: 1.8754428022340208
+    G10.47+0.03: 14.654313712932716
+    G10.62-0.38: 10.36368884187998
+    G12.81-0.20: 4.037590978170881
+    G13.18+0.06: 3.998842732080239
+    G13.66-0.60: 3.965098726041119
+    G14.11-0.57: 1.0980674058983302
+    G14.19-0.19: 5.9799020315281455
+    G14.49-0.14: 1.94278246727516
+    G14.63-0.58: 1.7565446299584897
+    G15.03-0.67: 1.9599610541713943
+    G18.61-0.07: 1.4529504405718856
+    G18.73-0.23: 3.946558643004384
+    G18.89-0.47: 3.87746848292546
+    G19.88-0.54: 2.725369130194149
+    G22.37+0.45: 2.2334635415722928
+    G23.21-0.38: 5.3215576604812105
+    G28.56-0.24: 2.00135317412325
+    G28.86+0.07: 1.6724447166657392
+    G30.85-0.08: 1.0384292597032818
+    G31.41+0.31: 7.634235223264329
+    G34.26+0.15: 11.79339165343781
+    G34.40+0.23: 4.893358882791944
+    G34.41+0.23: 6.975706242571226
+    G34.82+0.35: 0.6233212838879257
+    G35.20-0.74: 4.120487302502623
+    G37.55+0.20: 2.3647012203938407
+    G49.49-0.39: 30.638278402401056
+    G53.14+0.07: 1.2755372740021984
+    G59.78+0.07: 1.051817414971558
+  '88':
+    G08.71-0.41: 0.2555605423861423
+    G10.45-0.02: 0.5654586375969508
+    G10.47+0.03: 10.621236220690491
+    G10.62-0.38: 6.590559744325095
+    G12.81-0.20: 2.2391932908285748
+    G13.18+0.06: 1.043058241991689
+    G13.66-0.60: 1.388687476672136
+    G14.11-0.57: 0.2400487988472261
+    G14.19-0.19: 1.3734457205383674
+    G14.49-0.14: 0.1719527527259724
+    G14.63-0.58: 0.6227318765911359
+    G15.03-0.67: 0.9484169569327036
+    G18.61-0.07: 0.2409772077030058
+    G18.73-0.23: 1.43132088364319
+    G18.89-0.47: 1.240400088439042
+    G19.88-0.54: 0.940430197609148
+    G22.37+0.45: 0.4850629557476013
+    G23.21-0.38: 1.900341099965392
+    G28.56-0.24: 0.3811019169488633
+    G28.86+0.07: 0.8361276696375168
+    G30.85-0.08: 0.2353506461831835
+    G31.41+0.31: 6.015384520155143
+    G34.26+0.15: 7.861370828025846
+    G34.40+0.23: 1.3090143608612992
+    G34.41+0.23: 3.094135827250596
+    G34.82+0.35: 0.1891164402500769
+    G35.20-0.74: 1.7277506260977236
+    G37.55+0.20: 0.934902424028627
+    G49.49-0.39: 19.50272354704647
+    G53.14+0.07: 0.4187857086494883
+    G59.78+0.07: 0.4564477452483653
+probability_threshold: 0.37
+ratio_limits: {
+  '87-86': [ 0, 1.7 ],
+  '88-86': [ 0, 1.7 ],
+  '88-87': [ 0, 1.7 ],
+  '257-256': [ 0, 15 ],
+  '381-380': [ 0, 4 ],
+}
+ratios_to_include:
+- 87-86
+- 88-86
+- 88-87
+- 257-256
+- 381-380
+recompute_kde: true
+simulated_ratio_realizations: 1000
+use_model_for_inference: PLACEHOLDER

etl/config/ml_modelling.yml

+retrain: True
+use_model_for_training: constant_abundance_p15_q05
+
+overrides: {
+  'tdust_grid_type': 'linear',
+  'tdust_limits': [ 10, 35 ],
+  'tdust_step': 2.5,
+  'nh2_grid_type': 'log',
+  'nh2_limits': [ 1e3, 6.5e7 ],
+  'nh2_step': 1.316074013
+}
+
+#model_type: 'XGBoost'
+#model_parameters: {
+#  'n_estimators': 300,
+#  'max_depth': 4,
+#  'learning_rate': 0.02,
+#  'subsample': 0.5
+#}
+
+use_validation: True
+
+model_type: 'XGBoost_gridsearch'
+model_parameters: {
+  'param_grid':
+    {
+      'n_estimators': [ 300, 320, 50 ],
+      'max_depth': [ 5, 7, 2 ],
+      'learning_rate': [ 0.05, 0.1, 0.03 ],
+      'subsample': [ 1, 2, 1 ],
+      'lambda': [ 10, 20, 10 ],
+      'colsample_bynode': [ 0.75, 0.8, 0.1 ],
+#      'colsample_bylevel': [ 0.5, 1, 0.1 ],
+      'colsample_bytree': [ 0.75, 0.8, 0.2 ],
+      'min_child_weight': [ 100, 200, 100 ]
+    },
+  'param_gridsearch':
+    {
+      'cv': 5,
+      'n_jobs': 16
+    },
+}
+#model_parameters: {
+#  'param_grid':
+#    {
+#      'n_estimators': [ 400, 900, 100 ],
+#      'max_depth': [ 3, 7, 2 ],
+#      'learning_rate': [ 0.01, 0.06, 0.02 ],
+#      'subsample': [ 0.3, 0.9, 0.2 ]
+#    },
+#  'param_gridsearch':
+#    {
+#      'cv': 3,
+#      'n_jobs': 16
+#    },
+#}
+
+#model_type: 'auto_skl'
+#model_parameters: {
+#  'time_left_for_this_task': 3600,
+#  'per_run_time_limit': 600,
+#  'n_jobs': 12
+#}
+
+#model_type: 'RandomForest'
+#model_parameters: {
+#  'n_estimators': 20,
+#  'criterion': 'friedman_mse',
+#  'max_samples': 0.5
+#}
+

etl/credentials/example_credentials.yml

+# add a db_credentials.yml file with your credentials. The structure should be:
+#DB_USER: db_user B64-encoded
+#DB_PASS: db_pass B64-encoded
+#DB_HOST: db_host
+#DB_NAME: db_name
+#DB_PORT: db_port
\ No newline at end of file

etl/main.py

+import glob
+import os
+import uuid
+import sqlalchemy
+import argparse
+import sys
+from multiprocessing import Pool
+from stg.stg_build_db_structure import init_db, TmpExecutionQueue
+from itertools import product, chain
+from typing import Union, Tuple, Iterator
+from assets.commons import (cleanup_directory,
+                            setup_logger,
+                            validate_parameter)
+from assets.commons.parsing import parse_input_main
+from assets.commons.db_utils import upsert, get_pg_engine
+from stg.stg_radmc_input_generator import main as stg_main
+from mdl.mdl_execute_radmc_command import main as execute_radmc_script
+from prs.prs_compute_integrated_fluxes_and_ratios import main as prs_main
+from prs.prs_inspect_results import main as prs_inspection_main
+
+
+def compute_full_grid(tdust: float,
+                      nh2: float,
+                      line: int,
+                      density_keyword: str,
+                      dust_temperature_keyword: str) -> Tuple[float, float, int, str]:
+    """
+        Compute the full grid for a given dust temperature, hydrogen density, and line identifier, and return the results.
+
+        :param tdust: The dust temperature.
+        :param nh2: The H2 number density.
+        :param line: The line identifier for the RADMC-3D observation.
+        :param density_keyword: The keyword for the density in the grid configuration.
+        :param dust_temperature_keyword: The keyword for the dust temperature in the grid configuration.
+        :return: A tuple containing the dust temperature, H2 number density, line identifier, and the name of the
+            resulting FITS file.
+    """
+    scratch_dir = os.path.join('mdl', 'scratches', str(uuid.uuid4()))
+    stg_overrides = {
+        'grid': {
+            dust_temperature_keyword: tdust,
+            density_keyword: nh2,
+        }
+    }
+    overrides = {
+        'grid_lines': stg_overrides,
+        'model': {
+            'radmc_observation': {
+                'iline': line
+            }
+        }
+    }
+    tarname = stg_main(override_config=overrides,
+                       path_radmc_files=scratch_dir,
+                       run_id=run_id)
+    cube_fits_name = execute_radmc_script(grid_zipfile=tarname,
+                                          override_config=overrides,
+                                          radmc_input_path=scratch_dir,
+                                          run_id=run_id)
+    return tdust, nh2, line, cube_fits_name
+
+
+def initialize_queue(engine: sqlalchemy.engine,
+                     run_id: str,
+                     run_arguments: Iterator):
+    """
+        Initialize the execution queue for a specific run ID with given run arguments if not already initialized.
+
+        :param engine: The SQLAlchemy engine used to interact with the database.
+        :param run_id: The unique identifier for the run.
+        :param run_arguments: An iterator of run arguments, each containing dust temperature, density, line,
+            density keyword, and dust temperature keyword.
+        :return: None. Inserts entries into the execution queue if the queue is not already initialized.
+    """
+    is_initialized = engine.execute(f"select count(*) from tmp_execution_queue where run_id='{run_id}'").first()[0] != 0
+    if is_initialized is False:
+        for arguments in run_arguments:
+            raw_insert_entry = {'run_id': run_id,
+                                'dust_temperature': arguments[0],
+                                'density': arguments[1],
+                                'line': arguments[2],
+                                'density_keyword': arguments[3],
+                                'dust_temperature_keyword': arguments[4],
+                                'done': False}
+            upsert(
+                table_object=TmpExecutionQueue,
+                row_dict=raw_insert_entry,
+                conflict_keys=[
+                    TmpExecutionQueue.run_id,
+                    TmpExecutionQueue.dust_temperature,
+                    TmpExecutionQueue.density,
+                    TmpExecutionQueue.line,
+                    TmpExecutionQueue.density_keyword,
+                    TmpExecutionQueue.dust_temperature_keyword
+                ],
+                engine=engine
+            )
+
+
+def get_run_pars(engine: sqlalchemy.engine,
+                 run_id: str):
+    """
+        Get and mark the next pending row from the execution queue associated with the given run ID as done.
+
+        :param engine: The SQLAlchemy engine used to interact with the database.
+        :param run_id: The unique identifier for the run.
+        :return: The row corresponding to the next pending task in the execution queue not marked as done, or None if there
+            are no more pending tasks.
+    """
+    sql_query = sqlalchemy.text(f"""UPDATE tmp_execution_queue 
+                SET done = true 
+                WHERE row_id = (SELECT row_id 
+                                   FROM tmp_execution_queue
+                                   WHERE (run_id = '{run_id}')
+                                        AND (done is false)
+                                        AND pg_try_advisory_xact_lock(row_id) 
+                                   LIMIT 1 FOR UPDATE) 
+                RETURNING *""")
+    return engine.execution_options(autocommit=True).execute(sql_query).first()
+
+
+def verify_run(engine: sqlalchemy.engine,
+               run_id: str):
+    """
+        Verify the completion status of a run by resetting completed but unfinished tasks to pending.
+
+        :param engine: The SQLAlchemy engine used to interact with the database.
+        :param run_id: The unique identifier for the run.
+        :return: True if all completed tasks for the run have associated FITS cube names, indicating completion,
+            otherwise False.
+    """
+    sql_query = sqlalchemy.text(f"""UPDATE tmp_execution_queue 
+                SET done = false
+                WHERE row_id in (SELECT row_id 
+                                   FROM tmp_execution_queue
+                                   WHERE run_id = '{run_id}'
+                                        AND (done is true)
+                                        AND (fits_cube_name is null))""")
+    engine.execution_options(autocommit=True).execute(sql_query)
+    _remaining_models = compute_remaining_models(run_id=run_id)
+    return True if _remaining_models == 0 else False
+
+
+def insert_fits_name(engine: sqlalchemy.engine,
+                     row_id: int,
+                     fits_cube_name: str):
+    """
+        Insert the FITS cube name into the row of the execution queue with the specified row ID.
+
+        :param engine: The SQLAlchemy engine used to interact with the database.
+        :param row_id: The unique identifier for the row in the execution queue.
+        :param fits_cube_name: The name of the FITS cube associated with the row.
+        :return: None. Updates the row in the execution queue with the FITS cube name.
+    """
+    sql_query = sqlalchemy.text(f"""UPDATE tmp_execution_queue 
+                SET fits_cube_name = '{fits_cube_name}'
+                WHERE row_id = {row_id}""")
+    engine.execution_options(autocommit=True).execute(sql_query)
+
+
+def compute_grid_elements(run_id: str):
+    """
+        Compute grid elements for a given run ID by initializing the execution queue with parallel arguments.
+
+        :param run_id: The unique identifier for the run.
+        :return: None. Initializes the execution queue for the specified run ID.
+    """
+    init_db()
+    parallel_args, _ = get_parallel_args_and_nprocesses()
+    engine = get_pg_engine(logger=logger)
+    initialize_queue(engine=engine,
+                     run_id=run_id,
+                     run_arguments=parallel_args)
+    engine.dispose()
+
+
+def get_parallel_args_and_nprocesses() -> Tuple[Iterator, int]:
+    """
+        Get parallel computation arguments and the number of processes for computation.
+
+        :return: A tuple containing an iterator of parallel arguments and the number of processes to use.
+    """
+    _tdust_model_type, _model_type, dust_temperatures, densities, line_pairs, n_processes, _ = parse_input_main()
+    line_set = set(chain.from_iterable(line_pairs))
+    density_keyword = 'central_density' if _model_type == 'homogeneous' else 'density_at_reference'
+    dust_temperature_keyword = 'dust_temperature' if _tdust_model_type == 'isothermal' else 'dust_temperature_at_reference'
+    parallel_args = product(dust_temperatures, densities, line_set, [density_keyword], [dust_temperature_keyword])
+    return parallel_args, n_processes
+
+
+def compute_model(run_id: str):
+    """
+        Compute a model associated with a given run ID from parameters retrieved from the execution queue.
+
+        :param run_id: The unique identifier for the run.
+        :return: None. Computes a model and updates the database with the associated FITS cube name if parameters are
+            available in the execution queue.
+    """
+    engine = get_pg_engine(logger=logger)
+    parameters_set = get_run_pars(engine=engine,
+                                  run_id=run_id)
+    engine.dispose()
+    if parameters_set is not None:
+        _, _, _, fits_cube_name = compute_full_grid(tdust=parameters_set[2],
+                                                    nh2=parameters_set[3],
+                                                    line=parameters_set[4],
+                                                    density_keyword=parameters_set[5],
+                                                    dust_temperature_keyword=parameters_set[6])
+        engine = get_pg_engine(logger=logger)
+        insert_fits_name(engine=engine,
+                         row_id=parameters_set[0],
+                         fits_cube_name=fits_cube_name)
+        engine.dispose()
+    else:
+        logger.info('All models were completed.')
+
+
+def initialize_run():
+    """
+        Initialize a new run by generating a run ID if not provided, computing grid elements for the run,
+        and saving the run ID to a file for future reference.
+
+        :return: The generated or provided run ID.
+    """
+    if args.run_id is not None:
+        run_id = args.run_id
+    else:
+        logger.info('Generating new run_id')
+        run_id = str(uuid.uuid4())
+    compute_grid_elements(run_id=run_id)
+    sys.stdout.write(run_id)
+    with open('run_id.txt', 'w') as run_id_file:
+        run_id_file.write(f'{run_id}\n')
+    return run_id
+
+
+def compute_remaining_models(run_id: Union[None, str] = None) -> int:
+    """
+        Compute the number of pending models for a given run ID.
+
+        :param run_id: Optional. The unique identifier for the run. If None, it defaults to the value of the
+            'run_id' environment variable.
+        :return: The number of remaining models.
+    """
+    _run_id = validate_parameter(run_id, default=os.getenv('run_id'))
+    logger.info(_run_id)
+    sql_query = sqlalchemy.text(f"""SELECT count(*)
+                                    FROM tmp_execution_queue
+                                    WHERE (run_id = '{run_id}')
+                                        AND ((done is false)
+                                        OR ((done is true) AND (fits_cube_name is null)))""")
+    engine = get_pg_engine(logger=logger)
+    n_models = engine.execution_options(autocommit=True).execute(sql_query).first()[0]
+    engine.dispose()
+    sys.stdout.write(str(n_models))
+    return n_models
+
+
+def get_results(engine: sqlalchemy.engine,
+                run_id: str):
+    """
+        Retrieve the results of a given run ID from the execution queue.
+
+        :param engine: The SQLAlchemy engine used to interact with the database.
+        :param run_id: The unique identifier for the run.
+        :return: A list of tuples containing the dust temperature, density, line, and FITS cube name for each model
+            associated with the run ID.
+    """
+    sql_query = sqlalchemy.text(f"""SELECT dust_temperature
+                                           , density
+                                           , line
+                                           , fits_cube_name
+                                    FROM tmp_execution_queue
+                                    WHERE run_id = '{run_id}'""")
+    return engine.execution_options(autocommit=True).execute(sql_query).all()
+
+
+def cleanup_tmp_table(run_id: str,
+                      engine: sqlalchemy.engine):
+    """
+        Cleanup the temporary execution queue for a given run ID.
+
+        :param run_id: The unique identifier for the run.
+        :param engine: The SQLAlchemy engine used to interact with the database.
+        :return: None. Deletes all rows from the execution queue associated with the specified run ID.
+    """
+    sql_query = sqlalchemy.text(f"""DELETE
+                                    FROM tmp_execution_queue
+                                    WHERE run_id = '{run_id}'""")
+    return engine.execution_options(autocommit=True).execute(sql_query)
+
+
+def main_presentation_step(run_id: str,
+                           cleanup_scratches: bool = True,
+                           results_dict: Union[dict, None] = None) -> bool:
+    _tdust_model_type, _model_type, dust_temperatures, densities, line_pairs, n_processes, run_type = parse_input_main()
+
+    for folder in ('data', 'figures', 'trained_model'):
+        os.makedirs(os.path.join(
+            'prs',
+            'output',
+            'run_type',
+            run_type,
+            folder
+        ), exist_ok=True)
+
+    engine = get_pg_engine(logger=logger)
+    _results_dict = validate_parameter(results_dict,
+                                       default=get_results(engine=engine,
+                                                           run_id=run_id))
+
+    results_map = {}
+    for (tdust, nh2, line, cube_fits_name) in _results_dict:
+        results_map[f'{str(nh2)}_{str(tdust)}_{line}'] = cube_fits_name
+
+    for line_pair in line_pairs:
+        for tdust, nh2 in product(dust_temperatures, densities):
+            prs_main(cube_fits_list=[results_map[f'{str(nh2)}_{str(tdust)}_{line_pair[0]}'],
+                                     results_map[f'{str(nh2)}_{str(tdust)}_{line_pair[1]}']],
+                     run_id=run_id,
+                     engine=engine)
+
+    if cleanup_scratches is True:
+        scratches_dirs = glob.glob(os.path.join('mdl', 'scratches', '*'))
+        for scratches in scratches_dirs:
+            cleanup_directory(directory=scratches, logger=logger)
+
+    prs_inspection_main(run_id=run_id,
+                        is_isothermal=_tdust_model_type == 'isothermal',
+                        is_homogeneous=_model_type == 'homogeneous',
+                        engine=engine,
+                        run_type=run_type)
+    _run_success = verify_run(run_id=run_id,
+                              engine=engine)
+    if _run_success is True:
+        cleanup_tmp_table(run_id=run_id,
+                          engine=engine)
+    engine.dispose()
+    return _run_success
+
+
+def process_models(distributed: bool = False) -> Tuple[Union[None, dict], int]:
+    """
+        Process models either in a distributed environment or on a single machine (with multiprocessing).
+
+        :param distributed: A boolean flag indicating whether to process models in parallel. Defaults to False.
+        :return: A tuple containing results (if processed in parallel) and the number of remaining models.
+    """
+    if distributed is True:
+        compute_model(run_id=run_id)
+        results = None
+        remaining_models = compute_remaining_models(run_id)
+    else:
+        parallel_args, n_processes = get_parallel_args_and_nprocesses()
+        with Pool(n_processes) as pool:
+            results = pool.starmap(compute_full_grid, parallel_args)
+        remaining_models = 0
+    return results, remaining_models
+
+
+logger = setup_logger(name='MAIN')
+parser = argparse.ArgumentParser()
+parser.add_argument('--run_id')
+parser.add_argument('--cleanup_scratches')
+parser.add_argument('--distributed')
+args = parser.parse_args()
+
+if __name__ == '__main__':
+    run_id = initialize_run()
+    assert run_id is not None
+    _distributed = validate_parameter(args.distributed, default='false').lower() == 'true'
+
+    results, remaining_models = process_models(distributed=_distributed)
+    if remaining_models == 0:
+        logger.info('All grid points processed. Summarizing results.')
+        _cleanup = validate_parameter(args.cleanup_scratches,
+                                      default='true').lower() == 'true'
+        _run_success = main_presentation_step(run_id=run_id,
+                                              cleanup_scratches=_cleanup,
+                                              results_dict=results)
+        if (_run_success is True) or (_distributed is False):
+            logger.info('The run completed successfully!')
+        else:
+            logger.error('The run was incomplete. I have reset the done flag in the database for incomplete models')

etl/mdl/config/config.yml

+radmc_postprocessing:
+    nphot: 1000000
+    scattering_mode_max: 0
+    iranfreqmode: 1
+    tgas_eq_tdust: 1
+    lines_slowlvg_as_alternative: 1
+
+radmc_observation:
+    inclination: 0
+    position_angle: 0
+    iline: 2
+    width_kms: 10
+    nchannels: 100
+    npix: 101
+    threads: 12
+    image_size_pc: 2

etl/mdl/data/dustopac.inp

+2               Format number of this file
+1               Nr of dust species
+============================================================================
+1               Way in which this dust species is read
+0               0=Thermal grain
+silicate        Extension of name of dustkappa_***.inp file
+----------------------------------------------------------------------------

etl/mdl/mdl_execute_radmc_command.py

+import os
+import sys
+from itertools import product
+from typing import Union
+import sqlalchemy
+import shutil
+from astropy import units as u
+from datetime import datetime
+from radmc3dPy import image
+from stg.stg_build_db_structure import (LinePars,
+                                        SpeciesAndPartners,
+                                        ModelPars,
+                                        StarsPars)
+from assets.commons import (load_config_file,
+                            validate_parameter,
+                            setup_logger,
+                            compute_unique_hash_filename,
+                            get_value_if_specified,
+                            cleanup_directory)
+from assets.commons.parsing import read_abundance_variation_schema
+from assets.commons.db_utils import upsert, get_pg_engine
+from assets.constants import (radmc_options_mapping,
+                              radmc_lines_mode_mapping)
+
+logger = setup_logger(name='MDL')
+
+
+def save_cube_as_fits(cube_out_name: Union[str, None] = None,
+                      cube_out_path: Union[str, None] = None,
+                      path_radmc_files: Union[str, None] = None):
+    """
+    Convert the RADMC output to fits
+    :param cube_out_name: outfile name
+    :param cube_out_path: outfile path
+    :param path_radmc_files: path where the radmc input/output files are stored
+    """
+    _cube_out_name = validate_parameter(cube_out_name, default='test_cube.fits')
+    _cube_out_path = validate_parameter(cube_out_path, default=os.path.join('prs', 'fits', 'cubes'))
+    _path_radmc_files = validate_parameter(path_radmc_files, default=os.path.join('mdl', 'radmc_files'))
+    imdata = image.readImage(fname=os.path.join(_path_radmc_files, 'image.out'))
+    output_name = os.path.join(_cube_out_path, _cube_out_name)
+    if os.path.isfile(output_name):
+        os.remove(output_name)
+    imdata.writeFits(fname=output_name)
+
+
+def check_config(config: dict) -> bool:
+    """
+    Check if all mandatory keys are present in the configuration dictionary.
+    :param config: the configuration dictionary
+    :return: True is all keys are present, False otherwise
+    """
+    mandatory_keys = {
+        'inclination',
+        'position_angle',
+    }
+    return set(mandatory_keys).difference(set(config.keys())) == set()
+
+
+def get_command_options(config: dict) -> set:
+    """
+    Parse the radmc3d command options from the configuration dictionary
+    :param config: the configuration dictionary
+    :return: a set of string to postpone to the radmc3d image command
+    """
+    options_list = []
+    for key in config["radmc_observation"]:
+        try:
+            options_list.append(f'{radmc_options_mapping[key]} {config["radmc_observation"][key]}')
+        except KeyError:
+            pass
+    return set(options_list)
+
+
+def populate_model_table(config_mdl: dict,
+                         grid_zipfile: str,
+                         cube_filename: str,
+                         engine: sqlalchemy.engine,
+                         executed_on: datetime.timestamp,
+                         run_id: str):
+    """
+    Populate the model_parameters table in the DB
+    :param config_mdl: the model configuration
+    :param grid_zipfile: the name of the grid tarfile
+    :param cube_filename: the name of the fits cube
+    :param engine: the SQLAlchemy engine
+    :param executed_on: the timestamp of execution
+    :param run_id: the run unique identifier
+    """
+    model_pars_dict = {
+        'zipped_grid_name': grid_zipfile,
+        'fits_cube_name': cube_filename,
+        'nphotons': config_mdl['radmc_postprocessing']['nphot'],
+        'scattering_mode_max': int(config_mdl['radmc_postprocessing']['scattering_mode_max']),
+        'iranfreqmode': config_mdl['radmc_postprocessing']['iranfreqmode'],
+        'tgas_eq_tdust': config_mdl['radmc_postprocessing']['tgas_eq_tdust'],
+        'inclination': config_mdl['radmc_observation']['inclination'],
+        'position_angle': config_mdl['radmc_observation']['position_angle'],
+        'imolspec': get_value_if_specified(config_mdl['radmc_observation'], 'imolspec'),
+        'iline': get_value_if_specified(config_mdl['radmc_observation'], 'iline'),
+        'width_kms': get_value_if_specified(config_mdl['radmc_observation'], 'width_kms'),
+        'nchannels': get_value_if_specified(config_mdl['radmc_observation'], 'nchannels'),
+        'npix': get_value_if_specified(config_mdl['radmc_observation'], 'npix'),
+        'created_on': executed_on,
+        'run_id': run_id
+    }
+    upsert(
+        table_object=ModelPars,
+        row_dict=model_pars_dict,
+        conflict_keys=[ModelPars.fits_cube_name, ModelPars.run_id],
+        engine=engine
+    )
+
+
+def populate_species_and_partner_table(config_lines: dict,
+                                       engine: sqlalchemy.engine,
+                                       executed_on: datetime.timestamp,
+                                       grid_zipfile: str,
+                                       run_id: str):
+    """
+    Populate the species_and_partners table in the DB
+    :param config_lines: the line configuration
+    :param engine: the SQLAlchemy engine
+    :param executed_on: the timestamp of execution
+    :param grid_zipfile: the name of the grid tarfile
+    :param run_id: the run unique identifier
+    """
+    hot_core_specs = read_abundance_variation_schema(line_config=config_lines)
+    for (species, collision_partner) in product(config_lines['species_to_include'], config_lines['collision_partners']):
+        species_partner_dict = {
+            'zipped_grid_name': f'{grid_zipfile}',
+            'species_to_include': species,
+            'molecular_abundance': config_lines['molecular_abundances'][species],
+            'threshold': hot_core_specs[species]['threshold'],
+            'abundance_jump': hot_core_specs[species]['abundance_jump'],
+            'collision_partner': collision_partner,
+            'molecular_abundance_collision_partner': config_lines['molecular_abundances'][collision_partner],
+            'created_on': executed_on,
+            'run_id': run_id
+        }
+        upsert(
+            table_object=SpeciesAndPartners,
+            row_dict=species_partner_dict,
+            conflict_keys=[SpeciesAndPartners.zipped_grid_name,
+                           SpeciesAndPartners.species_to_include,
+                           SpeciesAndPartners.collision_partner,
+                           SpeciesAndPartners.run_id],
+            engine=engine
+        )
+
+
+def populate_line_table(config_lines: dict,
+                        engine: sqlalchemy.engine,
+                        executed_on: datetime.timestamp,
+                        grid_zipfile: str,
+                        run_id: str):
+    """
+    Populate the lines table in the DB
+    :param config_lines: the dictionary containing the line configuration
+    :param engine: the SQLAlchemy engine to use
+    :param executed_on: the timestamp of execution, to add to the record
+    :param grid_zipfile: the grid tarfile name, to be used as key
+    :param run_id: the run unique identifier
+    """
+    line_pars_dict = {
+        'zipped_grid_name': f'{grid_zipfile}',
+        'lines_mode': config_lines['lines_mode'],
+        'created_on': executed_on,
+        'run_id': run_id
+    }
+    upsert(
+        table_object=LinePars,
+        row_dict=line_pars_dict,
+        conflict_keys=[LinePars.zipped_grid_name, LinePars.run_id],
+        engine=engine
+    )
+
+
+def main(grid_zipfile: str,
+         run_id: str,
+         override_config: Union[dict, None] = None,
+         radmc_input_path: Union[str, None] = None,
+         engine: sqlalchemy.engine = None) -> str:
+    # This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files
+    # The reason for splitting the main input file from the rest is that some parameters can be changed
+    # independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset
+    _override_config = validate_parameter(override_config, default={'grid_lines': {}, 'model': {}})
+    executed_on = datetime.now()
+    _radmc_input_path = validate_parameter(radmc_input_path, default=os.path.join('mdl', 'radmc_files'))
+    config_stg = load_config_file(os.path.join('stg', 'config', 'config.yml'),
+                                  override_config=_override_config['grid_lines'])
+    config_lines = config_stg['lines']
+    config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'),
+                                  override_config=_override_config['model'])
+    assert check_config(config=config_mdl['radmc_observation'])
+    with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile:
+        outfile.write(f'cd {_radmc_input_path}\n')
+        options_set = get_command_options(config_mdl)
+        radmc_command = f'radmc3d image {" ".join(options_set)}'
+        outfile.write(radmc_command)
+
+    config_full = config_mdl.copy()
+    config_full.update(config_stg)
+    cube_filename = f'{compute_unique_hash_filename(config=config_full)}.fits'
+
+    # Execute radmc if not done already
+    if not os.path.isfile(os.path.join('prs', 'fits', 'cubes', cube_filename)):
+        logger.debug(f'Executing command: {radmc_command}')
+        execution_dir = os.getcwd()
+        os.chdir(_radmc_input_path)
+        os.system(radmc_command)
+        os.chdir(execution_dir)
+        logger.debug(f'Checking presence of file: {os.path.join(_radmc_input_path, "image.out")}')
+        assert os.path.isfile(os.path.join(_radmc_input_path, 'image.out'))
+
+        save_cube_as_fits(cube_out_name=cube_filename,
+                          cube_out_path=os.path.join('prs', 'fits', 'cubes'),
+                          path_radmc_files=radmc_input_path)
+    else:
+        logger.info('Computation performed already! Skipping...')
+
+    if engine is None:
+        engine = get_pg_engine(logger=logger, engine_kwargs={'pool_size': 2})
+
+    populate_line_table(config_lines=config_lines,
+                        engine=engine,
+                        executed_on=executed_on,
+                        grid_zipfile=grid_zipfile,
+                        run_id=run_id)
+
+    populate_species_and_partner_table(config_lines=config_lines,
+                                       engine=engine,
+                                       executed_on=executed_on,
+                                       grid_zipfile=grid_zipfile,
+                                       run_id=run_id)
+
+    populate_model_table(config_mdl=config_mdl,
+                         grid_zipfile=grid_zipfile,
+                         cube_filename=cube_filename,
+                         engine=engine,
+                         executed_on=executed_on,
+                         run_id=run_id)
+    engine.dispose()
+    return cube_filename
+
+
+if __name__ == '__main__':
+    main(grid_zipfile='459295aa894dffa8c521e606d14dbb6927638a2c.zip',
+         run_id='test_run')

etl/mdl/radmc_files/readme

+Here SAK will store the radmc files for each run.
\ No newline at end of file

etl/mdl/scratches/readme

+Folder that contains the scratch files used by SAK in order to avoid concurrency when running in parallel.
\ No newline at end of file

etl/prs/config/config.yml

+flux_computation:
+    central_frequency: 230.000
+    central_frequency_units: "GHz"
+    integration_limits: "all"
+    moment_zero_aggregation_function: 'sum'
+    aggregation_function: 'weighted_mean'
\ No newline at end of file

etl/prs/fits/cubes/readme

+In this directory you will find the postprocessed fits cubes.
\ No newline at end of file

etl/prs/fits/grids/readme

+In this directory you will find the fits files representing the model grid.
\ No newline at end of file

etl/prs/fits/moments/readme

+In this directory you will find the moment zero images of the lines.
\ No newline at end of file

etl/prs/fits/ratios/readme

+In this directory you will find the imagrs of the line ratios.
\ No newline at end of file

etl/prs/output/lte_approx/readme

+Here you will find the figure referring to the LTE column density approximation comparison.
\ No newline at end of file

etl/prs/output/readme

+In this directory you will find the final (summarized) output of the run(s).
\ No newline at end of file

etl/prs/output/run_type/readme

+Here you will find the results of the run, in the folder corresponding to the run type.
\ No newline at end of file