# star.jl : read star data

const STAR_P = Matrix{Float64}([
-0.90097 -0.43388
-0.90097 0.43388
-0.86451 -0.39443
-0.86051 0.3901
-0.84371 -0.42971
-0.84321 0.42967
-0.83242 -0.3597
-0.82 0.34626
-0.80909 0.38907
-0.79777 -0.32222
-0.79647 -0.42626
-0.78564 0.30909
-0.78504 0.42543
-0.78119 -0.37694
-0.76462 0.36736
-0.76098 -0.2824
-0.74803 -0.33296
-0.74789 0.26824
-0.74516 -0.42252
-0.73603 0.3214
-0.73588 0.42185
-0.7251 -0.24357
-0.72388 -0.3783
-0.71338 0.37276
-0.70751 -0.29252
-0.70711 0.22412
-0.7019 0.27417
-0.69493 -0.33601
-0.69103 -0.20671
-0.68995 -0.4185
-0.68588 0.32592
-0.68306 0.41799
-0.67067 -0.26099
-0.67052 0.18452
-0.66532 -0.37173
-0.65929 0.37056
-0.65747 0.27921
-0.65548 -0.16825
-0.64934 0.23178
-0.64892 -0.31472
-0.63935 -0.22039
-0.63575 -0.41454
-0.6342 0.14522
-0.6327 0.32338
-0.62951 0.41409
-0.61904 -0.12881
-0.61827 -0.27111
-0.61587 0.19149
-0.61573 -0.36309
-0.60899 0.27269
-0.60537 0.36672
-0.60508 -0.17994
-0.59709 0.10506
-0.5928 -0.31758
-0.58773 0.22856
-0.58637 -0.22848
-0.58409 -0.41078
-0.58363 0.1492
-0.58166 -0.088365
-0.58099 0.31782
-0.57615 0.4102
-0.57004 -0.13886
-0.56538 0.070752
-0.56303 -0.27407
-0.56217 -0.36309
-0.55741 0.18381
-0.55577 0.27079
-0.55277 -0.18618
-0.55266 0.36222
-0.54288 -0.0464
-0.54058 0.1249
-0.53676 -0.31795
-0.53485 -0.096479
-0.53262 0.035302
-0.53164 -0.23095
-0.5314 0.22464
-0.53051 -0.40687
-0.52792 0.31485
-0.52333 0.40634
-0.51889 -0.14291
-0.51102 0.08396
-0.50924 0.1724
-0.50734 -0.27411
-0.5072 -0.36071
-0.50253 0.26766
-0.50173 -0.049168
-0.5 6.1232e-17
-0.49973 0.35886
-0.49927 -0.18727
-0.48658 -0.096198
-0.48413 0.12923
-0.4807 -0.3161
-0.47988 0.21857
-0.47758 0.045037
-0.47671 -0.23021
-0.47618 -0.40291
-0.47496 0.3116
-0.47046 0.40249
-0.46772 -0.14124
-0.45927 -0.044641
-0.45451 0.174
-0.45375 0.088745
-0.45196 -0.35764
-0.45172 -0.27206
-0.45062 0.2637
-0.44684 0.35529
-0.44669 0.0016115
-0.44615 -0.18499
-0.4386 -0.09242
-0.42706 0.13181
-0.42561 0.21846
-0.42503 -0.31329
-0.42304 0.045802
-0.42241 0.30807
-0.42221 -0.2276
-0.42156 -0.39892
-0.41765 0.39864
-0.41682 -0.13817
-0.41227 -0.04467
-0.39878 0.17514
-0.39793 0.089063
-0.39694 0.26251
-0.3969 -0.35431
-0.39651 -0.26935
-0.39412 0.35173
-0.39366 0.001052
-0.39317 -0.18244
-0.3886 -0.091603
-0.3712 0.13202
-0.37057 0.21877
-0.36967 -0.31059
-0.36898 0.0449
-0.36884 0.30604
-0.3678 -0.22567
-0.36684 -0.39493
-0.36489 0.39479
-0.36487 -0.13701
-0.36285 -0.045128
-0.3436 0.1753
-0.34315 0.088222
-0.34228 0.26249
-0.34186 -0.3514
-0.34117 -0.26838
-0.34081 0.34897
-0.34068 -0.00013538
-0.33987 -0.1813
-0.33757 -0.091348
-0.31642 0.13155
-0.31585 0.21896
-0.3153 0.043673
-0.31408 0.30628
-0.31368 -0.22498
-0.31365 -0.31104
-0.3125 -0.13653
-0.31188 -0.045899
-0.31174 -0.39092
-0.31174 0.39092
-0.30442 0.43887
-0.30393 -0.44209
-0.29743 0.48462
-0.29655 -0.49034
-0.28935 0.53747
-0.28921 0.17521
-0.28916 0.087303
-0.28812 -0.54553
-0.28792 0.26298
-0.28778 -0.0013406
-0.28669 -0.26862
-0.28663 -0.18102
-0.28584 -0.091567
-0.28572 0.35044
-0.285 -0.35404
-0.28158 0.58835
-0.28038 -0.59624
-0.27409 0.63736
-0.27238 -0.64859
-0.26531 0.69482
-0.26446 -0.40743
-0.26368 -0.70555
-0.26338 0.40202
-0.2625 0.13109
-0.26202 0.042633
-0.2618 0.21931
-0.26011 0.30774
-0.2601 -0.046717
-0.26008 -0.22536
-0.25998 -0.13664
-0.25928 -0.31295
-0.25528 -0.76052
-0.25446 -0.51103
-0.25408 0.5011
-0.25388 0.7697
-0.24941 -0.45752
-0.2481 0.45088
-0.24713 -0.81383
-0.24581 0.82249
-0.24032 0.55182
-0.23919 -0.86583
-0.23815 0.87264
-0.23697 -0.56078
-0.23609 -0.36312
-0.23579 0.086624
-0.23557 0.17524
-0.23553 0.35662
-0.23504 -0.002297
-0.23441 0.26415
-0.23376 -0.092042
-0.23374 -0.18147
-0.23333 -0.27031
-0.23325 0.73325
-0.23252 0.65288
-0.23187 -0.91373
-0.23135 -0.61057
-0.23101 0.91936
-0.22887 -0.67028
-0.22531 0.60088
-0.22252 -0.97493
-0.22252 0.97493
-0.21581 -0.72508
-0.21535 -0.41209
-0.21367 0.40522
-0.21344 -0.50208
-0.21123 0.49821
-0.20927 0.13084
-0.20913 0.041902
-0.20884 -0.31844
-0.20874 0.31136
-0.20861 0.22001
-0.20801 -0.047429
-0.20761 -0.13716
-0.20736 -0.22668
-0.20668 0.78864
-0.20535 0.69446
-0.20304 -0.77766
-0.19828 -0.85007
-0.19566 -0.93569
-0.19378 -0.45624
-0.19335 0.54695
-0.19329 -0.54705
-0.19299 -0.63772
-0.19277 0.45063
-0.19265 0.84414
-0.19134 0.92939
-0.18583 -0.36612
-0.18479 0.64517
-0.1847 0.35875
-0.18293 0.086203
-0.18267 0.1755
-0.18251 -0.0029727
-0.1825 0.26618
-0.18219 -0.27359
-0.18166 -0.092601
-0.18144 -0.18245
-0.18083 0.74443
-0.17865 -0.68578
-0.17712 -0.59117
-0.17285 0.59105
-0.17014 -0.81263
-0.16952 -0.89752
-0.16476 -0.73334
-0.16447 0.89015
-0.16393 -0.50114
-0.1631 0.4968
-0.16304 -0.41166
-0.16172 0.40501
-0.1584 0.7994
-0.15796 -0.32039
-0.15734 0.31272
-0.15662 0.13077
-0.15656 0.041421
-0.15648 0.22102
-0.15592 -0.22862
-0.15591 -0.047964
-0.15557 -0.13786
-0.15483 0.69682
-0.14568 -0.63798
-0.14194 0.54211
-0.14192 -0.54781
-0.14166 -0.85683
-0.14128 -0.77297
-0.14108 0.63205
-0.13926 -0.45594
-0.13846 0.4501
-0.13543 0.84774
-0.13418 -0.3662
-0.13304 0.3589
-0.13183 0.75321
-0.13099 -0.2748
-0.13081 0.26691
-0.13049 0.17585
-0.13047 0.085931
-0.13019 -0.0033918
-0.12987 -0.18356
-0.12973 -0.093085
-0.12882 -0.68588
-0.12138 -0.59269
-0.11781 0.58427
-0.11549 0.6697
-0.11364 -0.5016
-0.1132 -0.81527
-0.11156 0.49489
-0.1103 -0.41122
-0.10925 0.4046
-0.10889 -0.72971
-0.10647 -0.32053
-0.10571 0.31275
-0.10534 0.8038
-0.10466 0.22121
-0.10455 -0.22931
-0.10444 0.13069
-0.10422 0.041171
-0.10392 -0.13842
-0.1039 -0.048317
-0.10016 0.71561
-0.095607 -0.63891
-0.089738 -0.54773
-0.087782 0.53858
-0.087699 0.62453
-0.085715 -0.45578
-0.084288 0.44933
-0.0841 -0.77277
-0.082161 -0.3658
-0.081172 0.35844
-0.079537 -0.27486
-0.079153 0.2666
-0.078577 0.17557
-0.078407 -0.18374
-0.078215 0.08586
-0.078032 -0.0036225
-0.077956 -0.093385
-0.076428 0.76157
-0.075891 -0.68438
-0.067551 -0.59284
-0.065297 0.67144
-0.063239 -0.50133
-0.061583 0.58009
-0.057965 0.49334
-0.0576 -0.41064
-0.056719 0.40375
-0.054748 -0.32021
-0.054084 0.31192
-0.053766 -0.72847
-0.053162 -0.22889
-0.052935 0.22033
-0.052269 -0.13843
-0.052248 0.13064
-0.052056 0.041098
-0.051973 -0.04852
-0.046193 0.71741
-0.043757 -0.63832
-0.037033 -0.54741
-0.035993 0.62533
-0.032291 -0.45497
-0.032165 0.53626
-0.030027 0.44817
-0.02997 -0.36538
-0.029393 0.35704
-0.028162 -0.2738
-0.027632 0.26471
-0.026778 -0.18334
-0.026327 0.17539
-0.026123 -0.093441
-0.026048 0.085884
-0.026009 -0.0037078
-0.022437 -0.68272
-0.014591 0.67126
-0.014242 -0.59313
-0.014037 -0.50051
-0.0072757 0.58106
-0.0046829 -0.4105
-0.0046299 0.40196
-0.0034727 0.49154
-0.0033654 -0.31831
-0.0029089 0.30833
-0.0014897 -0.22801
-0.00040952 -0.1383
-0.0004079 0.21995
-6.0753e-05 -0.048552
-7.7757e-06 0.1307
1.6295e-06 0.041118
0.009434 -0.63617
0.01588 -0.55032
0.016596 0.62571
0.020993 0.35079
0.021249 -0.36227
0.021936 -0.45567
0.02211 0.53798
0.023678 -0.27229
0.024397 0.44559
0.025221 -0.18305
0.025465 0.26411
0.025783 -0.093389
0.025956 -0.003685
0.026059 0.085988
0.026133 0.17549
0.030418 -0.50773
0.041342 -0.58957
0.043864 0.39161
0.046026 -0.4054
0.047708 0.58028
0.048706 0.49581
0.04874 -0.3159
0.050804 -0.22763
0.051056 0.30769
0.051577 -0.13819
0.051863 -0.048508
0.051998 0.041203
0.052218 0.1309
0.052529 0.22012
0.068437 -0.4902
0.073579 -0.35847
0.073981 -0.54191
0.074378 0.42014
0.074718 -0.44356
0.076152 0.35076
0.076588 -0.27192
0.076704 0.46184
0.077391 -0.18292
0.077741 -0.093325
0.077899 -0.0036247
0.078102 0.086145
0.078611 0.17582
0.079154 0.26442
0.079351 0.53406
0.10022 -0.39761
0.1027 -0.31568
0.10326 -0.22756
0.10363 -0.13812
0.10381 -0.048464
0.10395 0.041289
0.10435 0.13116
0.10552 0.30836
0.10555 0.22067
0.10619 0.38702
0.11126 -0.48746
0.11126 0.48746
0.12631 -0.43772
0.12668 0.43406
0.12961 -0.18289
0.1297 -0.27201
0.12974 -0.093294
0.12984 -0.0036
0.12996 -0.35761
0.13007 0.086268
0.13094 0.17628
0.13299 0.26521
0.13454 0.34881
0.15567 -0.22765
0.15572 -0.13811
0.1558 -0.048482
0.15588 0.041283
0.15633 0.13137
0.15721 -0.40006
0.15737 -0.31546
0.15778 0.46918
0.15844 0.22141
0.15897 -0.51488
0.15919 -0.46995
0.15936 0.51511
0.15971 0.39543
0.16154 0.30783
0.18181 -0.093335
0.18183 -0.003708
0.18185 -0.18291
0.18195 0.086236
0.18294 0.17668
0.18297 -0.2721
0.1851 -0.359
0.18764 0.26544
0.18778 0.43678
0.18819 -0.43823
0.18824 0.35365
0.20344 -0.54044
0.20398 0.54075
0.20659 0.48958
0.20696 -0.49083
0.20783 0.041052
0.20788 -0.13814
0.20788 -0.048658
0.20796 -0.22771
0.20809 0.13136
0.21133 0.2218
0.21149 -0.31652
0.21467 -0.40095
0.21528 0.3107
0.21563 0.39831
0.23377 0.085875
0.23391 -0.0040934
0.23403 -0.093519
0.23416 -0.18283
0.23456 0.17672
0.23564 -0.27294
0.2377 -0.45236
0.23805 0.45033
0.24026 -0.36015
0.24133 0.26726
0.24227 0.35594
0.244 -0.56375
0.24473 0.56417
0.25337 -0.50665
0.25381 0.50523
0.25959 -0.22797
0.25966 0.13082
0.25983 0.040411
0.26015 -0.049131
0.26024 -0.13815
0.2631 0.22287
0.26517 -0.40919
0.26559 -0.31784
0.26578 0.40561
0.26912 0.31201
0.2851 0.17665
0.28559 0.084787
0.28571 -0.4598
0.28616 -0.0049461
0.28653 -0.18277
0.2866 -0.093714
0.28664 0.45681
0.28748 -0.27389
0.2884 -0.54183
0.28846 0.54119
0.28964 -0.58997
0.29094 0.59072
0.29204 -0.36508
0.29362 0.35887
0.29549 0.26811
0.31033 -0.2283
0.31076 0.12958
0.311 -0.50228
0.31201 0.038627
0.31214 0.50018
0.31282 -0.13809
0.31288 -0.049423
0.31379 0.2232
0.31507 0.40688
0.31545 -0.41294
0.31814 -0.32086
0.32239 0.31264
0.33051 -0.61346
0.33207 0.61436
0.3344 0.17551
0.33637 -0.55602
0.33716 0.082063
0.33719 0.55324
0.33754 -0.27605
0.33889 -0.45788
0.33906 -0.18266
0.3393 -0.0062857
0.33943 0.45187
0.33951 -0.093042
0.34411 -0.36656
0.34484 0.35812
0.3505 0.26774
0.35927 -0.2292
0.35996 -0.50815
0.361 0.12586
0.36211 0.4981
0.36249 0.2223
0.36497 0.036381
0.36517 -0.13624
0.36621 0.40462
0.36685 -0.049946
0.36781 -0.41073
0.37191 -0.32199
0.37424 -0.59139
0.37451 0.58857
0.37624 0.31017
0.37639 -0.63983
0.37834 0.64095
0.38138 0.17035
0.38633 -0.27703
0.38812 -0.45858
0.38908 0.53488
0.38947 0.079671
0.39066 -0.17912
0.39272 0.45156
0.3929 -0.55149
0.39354 -0.0078401
0.39389 -0.092765
0.3957 0.3566
0.39825 -0.36427
0.40414 -0.22276
0.40663 0.21532
0.40755 0.26274
0.40999 -0.50491
0.41241 0.12291
0.41629 0.40545
0.41791 -0.66369
0.41852 0.035961
0.41856 -0.41178
0.41917 -0.13542
0.42029 0.49168
0.42049 0.56891
0.42211 -0.051552
0.42233 -0.60669
0.42416 0.61745
0.42503 0.66778
0.42837 0.30823
0.42843 -0.31834
0.42903 -0.26547
0.42933 0.1674
0.43909 -0.46004
0.44105 -0.55522
0.44325 0.07943
0.44474 -0.17693
0.44476 0.35705
0.44743 -0.0063528
0.4494 0.52837
0.44989 -0.096828
0.45017 -0.36762
0.45048 -0.21694
0.45048 0.21694
0.45098 0.45008
0.45746 -0.51029
0.45938 -0.68752
0.4648 0.2637
0.46501 0.12265
0.46502 0.40598
0.46725 0.58084
0.46739 0.69212
0.4682 -0.63389
0.46872 -0.2765
0.46943 -0.41992
0.4731 0.039431
0.47328 0.63303
0.47457 -0.58663
0.47492 -0.050802
0.47492 0.16872
0.47591 -0.14395
0.47953 0.31179
0.48182 0.48582
0.48424 -0.32717
0.48919 -0.47473
0.4941 0.35935
0.49848 -0.53425
0.49908 0.085807
0.49991 0.53645
0.50022 -0.0047965
0.50058 -0.38051
0.50197 -0.095032
0.5022 -0.19653
0.50484 -0.71365
0.50517 -0.6667
0.50642 0.66989
0.50658 0.1948
0.50897 0.4079
0.51369 0.58833
0.51646 0.72032
0.51735 0.13658
0.51741 -0.43547
0.51859 -0.58406
0.52209 0.45074
0.52229 -0.14157
0.52562 0.040312
0.52804 -0.047559
0.52986 -0.63206
0.53186 0.63371
0.53478 -0.49217
0.53653 0.49789
0.54707 -0.73792
0.54997 0.087741
0.55077 -0.54439
0.55151 0.54681
0.55293 0.68943
0.55333 -0.68708
0.55383 -0.090598
0.5545 -0.0031196
0.55463 0.17582
0.55477 -0.17577
0.56336 0.74728
0.5662 0.59477
0.56741 -0.59871
0.568 0.1289
0.57323 -0.1287
0.57751 0.047381
0.5815 0.64473
0.58307 -0.050436
0.58365 -0.65176
0.5863 -0.76046
0.59702 0.6954
0.60147 -0.70994
0.60429 -0.00060389
0.60766 0.15489
0.60832 0.098195
0.60994 -0.15399
0.61055 0.73957
0.61463 -0.096944
0.62349 -0.78183
0.62349 0.78183
0.62485 0.049113
0.63357 -0.04583
0.65437 0.13645
0.65517 0.0032417
0.65801 -0.13501
0.66114 0.090053
0.66771 -0.082749
0.67516 0.045181
0.68649 -0.030527
0.70693 0.1157
0.70879 -0.11497
0.71456 0.018926
0.71778 0.070637
0.72067 -0.065609
0.73873 -0.016611
0.76121 -0.09427
0.76204 0.093945
0.76516 0.042919
0.77593 -0.047034
0.78493 0.0029256
0.81349 0.073631
0.81492 -0.073069
0.82671 0.028963
0.8287 -0.022385
0.86619 -0.052826
0.87162 0.050683
0.87833 0.001808
0.91453 -0.033742
0.92781 0.028498
0.95816 -0.016518
1 -2.4493e-16
])

const STAR_T = Matrix{Int64}([
44 36 31
2 4 6
32 36 45
657 676 664
664 644 657
689 670 686
1 5 3
9 4 8
13 6 9
9 6 4
18 20 12
24 20 31
31 36 24
24 36 32
32 21 24
680 664 676
670 664 680
680 686 670
720 718 719
421 407 430
451 465 444
444 430 451
444 452 431
465 452 444
431 421 444
421 430 444
165 190 161
200 190 165
311 270 292
292 329 311
294 330 313
329 292 313
348 330 362
329 313 348
348 313 330
688 681 679
615 642 643
665 681 678
665 643 642
665 678 656
656 643 665
5 11 7
7 3 5
15 12 20
20 24 15
15 9 8
8 12 15
15 24 21
15 21 13
13 9 15
19 30 23
241 265 283
283 265 303
173 197 216
257 216 238
238 216 197
192 177 210
628 645 612
649 645 628
713 711 714
671 676 657
657 647 671
717 719 718
599 622 600
329 348 364
512 503 528
466 452 481
481 452 465
408 421 431
431 422 408
291 270 311
174 200 165
217 236 212
212 236 259
37 44 31
203 181 224
224 181 202
221 180 204
221 265 241
227 204 184
660 665 642
660 679 681
681 665 660
615 643 621
621 643 656
656 651 621
606 618 587
587 618 613
487 481 465
503 481 487
556 574 586
586 570 556
530 504 515
14 17 10
10 7 14
14 7 11
14 23 17
14 11 19
19 23 14
17 23 28
173 216 175
162 160 191
162 197 173
191 197 162
191 160 194
194 221 241
180 221 194
223 197 191
223 238 197
223 194 241
191 194 223
277 297 257
257 238 277
648 664 670
644 664 648
648 620 644
214 243 218
214 261 243
214 199 261
284 266 307
287 266 254
331 307 287
287 307 266
261 199 242
284 261 242
242 266 284
199 196 242
245 216 257
606 598 630
630 598 616
705 711 708
708 713 709
711 713 708
646 652 613
613 618 646
712 707 709
712 710 707
715 710 712
709 713 712
613 652 631
663 671 647
647 637 663
715 712 716
716 712 713
718 715 716
716 717 718
716 713 714
714 717 716
615 621 588
588 621 595
614 603 635
635 651 662
622 599 632
534 553 526
558 553 534
503 512 491
491 481 503
466 481 491
491 482 466
394 407 421
421 408 394
240 255 276
57 77 65
27 37 31
31 20 27
27 20 18
96 116 103
84 77 96
96 103 84
84 103 92
84 65 77
181 203 163
224 202 247
247 269 224
270 291 247
247 291 269
393 364 378
378 364 348
378 348 362
362 392 378
393 378 406
406 378 392
420 430 407
420 407 393
393 406 420
249 292 270
167 205 182
362 330 345
141 151 133
206 227 184
204 227 246
246 221 204
265 221 246
626 621 651
635 603 626
626 651 635
626 603 595
595 621 626
435 437 414
558 570 583
526 553 538
538 553 567
511 496 526
485 496 511
526 538 511
511 538 525
552 538 567
525 538 552
513 487 504
504 530 513
528 503 513
503 487 513
574 556 543
543 513 530
543 556 528
528 513 543
649 628 617
617 636 649
35 23 30
35 28 23
35 30 42
98 79 88
88 106 98
241 283 263
263 223 241
238 223 263
263 277 238
331 287 314
349 331 314
266 242 232
232 242 196
232 196 192
254 266 232
192 210 232
232 210 254
254 210 233
233 210 177
211 175 216
216 245 211
245 233 211
177 175 211
211 233 177
257 297 281
281 245 257
684 690 695
675 690 684
682 690 675
687 694 696
629 605 641
668 645 649
649 673 668
644 620 625
657 644 625
625 647 657
593 596 566
597 625 620
596 625 597
566 596 569
569 540 566
596 597 569
634 618 606
634 646 618
606 630 634
659 663 637
659 631 652
637 631 659
595 603 577
577 565 595
555 542 577
577 542 565
382 351 367
397 382 367
412 382 397
595 565 568
568 588 595
696 694 700
672 662 651
656 678 672
672 651 656
685 687 674
694 687 685
610 632 599
526 496 507
507 534 526
512 534 507
507 496 482
482 491 507
507 491 512
541 534 512
541 512 528
541 570 558
558 534 541
528 556 541
541 556 570
380 364 393
393 407 380
407 394 380
379 394 408
346 379 361
359 325 344
256 240 276
239 200 256
219 179 189
176 179 215
255 240 215
179 219 215
215 219 255
47 41 33
10 17 16
29 33 41
142 135 156
44 60 51
51 61 45
36 44 51
51 45 36
44 37 50
50 60 44
114 106 97
97 88 78
106 88 97
101 111 93
116 135 123
123 103 116
135 142 123
120 111 101
248 203 224
224 269 248
225 247 202
202 182 225
225 182 205
205 249 225
270 247 225
225 249 270
590 610 581
581 610 599
571 599 600
492 480 467
467 480 448
502 527 516
502 480 492
492 519 502
502 519 527
428 441 419
419 405 428
419 406 392
392 405 419
430 420 442
451 430 442
442 463 451
504 487 477
451 463 477
477 465 451
477 487 465
515 504 488
463 479 488
504 477 488
488 477 463
292 249 273
294 313 273
273 313 292
164 202 181
164 182 202
362 345 376
376 392 362
376 405 392
390 405 376
312 330 294
312 345 330
439 427 448
426 443 454
358 375 343
390 375 403
345 312 327
360 375 390
390 376 360
360 376 345
345 327 360
343 375 360
360 327 343
141 133 122
122 133 114
180 194 158
158 194 160
98 106 117
204 180 171
184 204 171
171 151 184
114 133 125
136 117 125
125 106 114
125 117 106
183 163 203
227 206 250
250 271 289
268 246 227
268 250 289
227 250 268
195 198 235
259 279 235
235 212 259
198 212 235
410 412 435
435 414 410
425 414 437
601 583 570
601 570 586
601 628 612
612 583 601
586 617 601
601 617 628
567 553 584
584 598 567
584 553 558
558 583 584
616 598 584
612 616 584
584 583 612
341 325 359
606 587 580
580 552 567
580 598 606
567 598 580
576 587 613
525 552 535
54 64 47
69 88 79
69 51 60
69 60 78
78 88 69
69 79 61
61 51 69
254 233 275
298 314 275
275 233 245
275 287 254
275 314 287
275 281 298
245 281 275
684 673 661
661 675 684
649 636 661
661 673 649
701 707 702
696 698 691
691 687 696
705 708 703
703 700 705
703 698 696
696 700 703
629 641 653
611 641 605
600 622 611
654 661 636
675 661 654
636 624 654
654 624 638
677 691 682
687 691 677
699 695 702
619 625 596
596 593 619
647 625 619
619 637 647
613 631 592
592 576 613
557 576 592
683 673 684
683 668 673
603 614 585
585 577 603
555 577 585
614 602 585
381 397 367
381 350 365
367 350 381
332 342 365
365 350 332
332 350 315
539 568 565
539 542 522
565 542 539
590 602 623
623 610 590
633 602 614
633 614 635
650 623 633
633 623 602
572 602 590
572 585 602
572 546 555
555 585 572
650 633 658
635 662 658
658 633 635
669 653 641
669 685 674
674 653 669
363 380 394
394 379 363
379 346 363
308 344 325
308 325 289
289 271 308
315 350 333
367 351 333
333 350 367
213 256 200
240 256 213
200 174 213
213 174 176
176 215 213
213 215 240
189 195 234
234 219 189
195 235 234
65 84 72
72 54 65
64 54 72
72 84 92
56 41 47
47 64 56
64 75 56
127 108 115
37 27 39
39 50 37
55 50 39
17 28 25
25 16 17
41 56 52
46 52 62
101 93 82
105 93 111
114 97 105
105 122 114
111 122 105
76 55 66
66 82 76
76 82 93
131 123 142
92 103 112
103 123 112
123 131 112
119 138 126
145 155 167
145 138 155
132 126 145
145 126 138
228 248 271
271 250 228
228 250 206
203 248 228
228 183 203
206 183 228
508 514 536
536 514 546
524 551 537
524 495 508
508 536 524
524 536 551
590 581 563
581 551 563
563 572 590
551 536 563
546 572 563
563 536 546
564 581 599
599 571 564
564 551 581
537 551 564
454 443 468
537 519 509
509 524 537
495 524 509
509 519 492
545 519 537
537 564 545
545 564 571
571 554 545
527 519 545
545 554 527
448 480 464
464 439 448
480 502 490
490 464 480
478 464 490
490 502 516
429 419 441
429 442 420
429 420 406
406 419 429
463 442 450
441 462 450
450 429 441
442 429 450
450 479 463
450 462 479
274 312 294
172 142 156
231 272 253
448 427 440
440 426 454
440 467 448
454 467 440
388 403 375
375 358 388
402 426 416
416 388 402
403 388 416
427 403 416
416 440 427
426 440 416
418 403 427
428 405 418
418 405 390
390 403 418
418 439 428
427 439 418
343 327 309
272 288 309
253 272 293
293 327 312
272 309 293
293 309 327
293 274 253
312 274 293
151 171 144
144 133 151
144 125 133
136 125 144
166 151 141
184 151 166
166 206 184
166 183 206
246 268 286
303 265 286
265 246 286
396 382 412
412 410 396
395 408 422
395 379 408
361 379 395
423 391 409
422 433 409
409 433 423
409 395 422
445 452 466
431 452 445
445 422 431
445 433 422
466 482 456
456 445 466
433 445 456
549 576 557
65 54 49
49 35 42
42 57 49
57 65 49
698 701 693
693 690 682
682 691 693
693 691 698
709 707 704
707 701 704
704 708 709
704 703 708
704 701 698
698 703 704
629 653 640
636 617 604
604 624 636
604 589 624
574 589 604
586 574 604
604 617 586
667 677 682
667 654 638
682 675 667
675 654 667
706 699 702
702 707 706
707 710 706
557 592 573
573 593 566
637 619 608
608 619 593
608 631 637
608 592 631
593 573 608
608 573 592
695 699 692
684 695 692
692 683 684
276 255 295
295 315 276
295 332 315
632 610 639
610 623 639
639 623 650
328 363 346
328 291 311
290 308 271
290 248 269
271 248 290
276 315 296
315 333 296
296 256 276
321 342 304
342 332 304
332 295 304
258 235 279
258 234 235
258 279 300
115 108 95
75 95 89
108 99 89
89 95 108
146 127 134
143 152 134
134 152 146
134 127 115
118 99 108
108 127 118
186 231 208
208 231 253
185 205 167
167 155 185
71 82 66
16 25 22
33 29 22
22 25 33
52 46 38
38 29 41
41 52 38
67 50 55
55 76 67
78 60 67
60 50 67
93 105 85
85 76 93
85 105 97
85 67 76
85 97 78
78 67 85
124 112 131
124 134 115
124 131 143
143 134 124
94 81 74
128 138 119
146 152 169
169 154 146
169 152 186
187 154 169
186 208 169
169 208 187
132 145 150
150 145 167
167 182 150
182 164 150
149 139 163
149 166 141
163 183 149
183 166 149
120 139 130
111 120 130
130 122 111
141 122 130
130 149 141
139 149 130
163 139 148
181 163 148
148 164 181
110 120 101
129 139 120
120 110 129
129 110 121
129 148 139
555 546 529
546 514 529
529 514 500
529 542 555
457 476 458
458 437 435
435 457 458
500 514 493
493 476 500
493 514 508
473 498 500
500 476 473
473 476 457
453 468 443
453 425 437
443 425 453
483 467 454
454 468 483
492 467 483
483 468 495
483 509 492
495 509 483
547 554 575
516 527 547
527 554 547
575 554 582
605 575 582
582 611 605
600 611 582
582 571 600
582 554 571
439 464 449
449 462 441
449 464 478
478 462 449
449 441 428
428 439 449
479 462 489
489 462 478
142 172 153
143 131 153
153 131 142
222 200 239
190 200 222
302 282 319
251 288 272
272 231 251
251 226 267
267 288 251
402 385 411
411 426 402
443 426 411
411 425 443
157 144 171
180 158 157
157 171 180
136 144 157
306 286 268
306 268 289
289 325 306
325 341 306
351 382 368
382 396 368
415 384 404
404 391 423
404 432 415
423 432 404
371 339 357
357 384 371
359 391 374
374 341 359
391 404 374
374 404 384
357 341 374
374 384 357
361 395 377
377 344 361
377 409 391
395 409 377
359 344 377
377 391 359
469 456 482
485 461 469
469 496 485
482 496 469
461 432 446
446 469 461
456 469 446
433 456 446
423 433 446
446 432 423
349 314 334
334 314 298
297 277 317
561 549 535
561 535 552
587 576 561
576 549 561
561 580 587
552 580 561
383 369 400
352 369 383
531 549 557
40 49 54
40 54 47
28 35 40
35 49 40
47 33 40
40 25 28
33 25 40
697 701 702
697 693 701
702 695 697
697 695 690
690 693 697
629 640 609
591 575 605
591 609 579
591 605 629
629 609 591
677 667 655
655 667 638
559 589 574
574 543 559
559 543 530
363 328 347
329 364 347
311 329 347
347 328 311
364 380 347
380 363 347
291 328 310
310 328 346
269 291 310
310 290 269
344 308 326
308 290 326
290 310 326
326 310 346
326 346 361
361 344 326
319 282 299
219 234 260
255 219 260
260 295 255
260 304 295
83 95 75
83 72 92
64 72 83
83 75 64
92 112 104
104 83 92
95 83 104
115 95 104
104 124 115
112 124 104
75 89 68
62 52 68
68 56 75
68 52 56
170 154 187
187 207 170
170 207 185
170 185 155
229 249 205
205 185 229
185 207 229
229 273 249
86 73 90
90 100 86
80 89 99
99 90 80
80 90 73
80 73 62
62 68 80
80 68 89
59 46 62
62 73 59
58 71 66
119 126 107
107 100 119
155 138 147
138 128 147
147 170 155
154 170 147
137 128 118
137 127 146
137 118 127
146 154 137
154 147 137
137 147 128
91 110 101
101 82 91
91 71 81
82 71 91
148 129 140
132 150 140
140 121 132
140 129 121
140 150 164
164 148 140
500 498 520
520 529 500
520 498 522
522 542 520
542 529 520
471 458 476
476 493 471
437 458 471
471 453 437
516 547 532
517 489 506
506 490 516
478 490 506
506 489 478
516 532 506
506 532 517
505 488 479
479 489 505
515 488 505
505 489 517
505 533 515
517 533 505
178 172 156
178 201 172
178 156 159
267 226 244
226 201 244
168 152 143
143 153 168
186 152 168
286 306 323
323 339 303
303 286 323
357 339 323
323 341 357
323 306 341
339 371 355
398 384 415
398 371 384
415 432 447
447 432 461
472 461 485
472 447 461
460 447 472
352 334 318
318 334 298
318 281 297
298 281 318
277 263 301
301 317 277
301 263 283
349 334 366
366 334 352
352 383 366
549 531 518
535 549 518
501 499 474
494 518 501
501 518 531
557 573 544
544 531 557
544 573 566
566 540 544
540 523 544
627 609 640
627 655 638
640 655 627
677 655 666
674 687 666
687 677 666
666 655 640
666 653 674
666 640 653
548 559 530
548 530 515
515 533 548
262 222 239
262 299 282
351 368 335
319 299 335
316 296 333
316 335 299
316 333 351
351 335 316
234 258 280
280 260 234
280 258 300
304 260 280
280 300 321
321 304 280
187 208 230
230 207 187
253 274 230
230 208 253
99 118 109
109 90 99
118 128 109
100 90 109
119 100 109
109 128 119
74 81 63
81 71 63
53 58 43
71 58 53
53 63 71
70 73 86
70 59 73
43 58 48
55 39 48
66 55 48
48 58 66
39 34 48
48 34 43
26 27 18
26 39 27
26 34 39
94 107 113
113 107 126
113 126 132
132 121 113
87 107 94
87 94 74
86 100 87
100 107 87
87 70 86
110 91 102
94 113 102
81 94 102
102 91 81
121 110 102
102 113 121
484 493 508
484 471 493
508 495 484
495 468 484
468 453 484
453 471 484
562 548 533
579 562 550
550 562 533
550 533 517
517 532 550
560 547 575
560 532 547
560 550 532
579 550 560
575 591 560
560 591 579
190 222 193
193 178 159
159 161 193
193 161 190
267 244 285
282 302 264
302 285 264
264 285 244
226 251 209
209 251 231
209 231 186
186 168 209
188 153 172
188 168 153
188 209 168
226 209 188
188 201 226
172 201 188
358 340 373
373 385 402
402 388 373
373 388 358
324 340 358
324 309 288
324 358 343
343 309 324
319 335 353
353 335 368
385 370 399
399 411 385
414 425 399
425 411 399
322 285 302
497 472 485
485 511 497
497 511 525
474 459 475
475 501 474
494 501 475
470 494 475
372 401 387
387 354 372
369 354 387
400 369 387
387 424 400
387 401 424
389 401 372
372 355 389
389 355 371
371 398 389
434 398 415
415 447 434
434 447 460
424 401 436
352 318 336
317 354 336
297 317 336
336 318 297
336 369 352
336 354 369
337 355 372
372 354 337
317 301 337
337 354 317
521 544 523
531 544 521
523 499 521
499 501 521
521 501 531
607 627 638
638 624 607
624 589 607
579 609 594
609 627 594
627 607 594
594 562 579
296 316 278
278 262 239
299 262 278
278 316 299
239 256 278
256 296 278
207 230 252
273 229 252
252 229 207
294 273 252
252 274 294
252 230 274
178 193 220
201 178 220
220 244 201
220 264 244
370 353 386
386 368 396
386 353 368
386 410 414
386 396 410
414 399 386
386 399 370
340 322 356
356 370 385
385 373 356
356 373 340
338 353 370
370 356 338
338 356 322
338 322 302
338 302 319
319 353 338
267 285 305
285 322 305
305 322 340
305 288 267
305 324 288
340 324 305
535 518 510
510 518 494
525 535 510
510 497 525
472 497 486
470 460 486
460 472 486
497 510 486
486 494 470
486 510 494
459 436 455
470 475 455
455 475 459
401 389 417
417 436 401
339 355 320
355 337 320
303 339 320
320 337 301
320 283 303
320 301 283
578 607 589
578 594 607
562 594 578
589 559 578
559 548 578
548 562 578
237 193 222
237 220 193
237 262 282
222 262 237
282 264 237
264 220 237
413 389 398
413 417 389
398 434 413
438 455 436
436 417 438
417 413 438
438 413 434
438 434 460
438 460 470
470 455 438
])

star() = (STAR_P,STAR_T)


import ArgParse, Gurobi, DataFrames
using ArgCheck

const arg_parse_settings = ArgParse.ArgParseSettings(prog="var", fromfile_prefix_chars='@',
                                                     description="Calculates the steady state as function of ξ.")

@ArgParse.add_arg_table arg_parse_settings begin
    "--sto"
        help = "stoichiometric matrix"
        arg_type = String
        required = true
    "--mets"
        help = "metabolites"
        arg_type = String
        required = true
    "--rxns"
        help = "reactions"
        arg_type = String
        required = true
    "--A"
        help = "biomass synthesis cost"
        arg_type = Float64
        default = 0.
        range_tester = A -> 0 ≤ A < Inf
    "--c"
        help = "media"
        arg_type = String
        required = true
    "--xi"
        help = "File with a list of ξ values to use. Execution stops at the first infeasible value of ξ found."
        arg_type = String
        required = true
    "--out"
        help = "output file"
        arg_type = String
        required = true
    "--prefix"
        help = "all paths arguments (--sto, --mets, --rxns, --c, --xi, --out) will be prefixed with this string."
        arg_type = String
        default = ""
end

const args = ArgParse.parse_args(arg_parse_settings)

set_multiobj_n!(gurobi_model::Gurobi.Model, n::Int) = Gurobi.set_intattr!(gurobi_model, "NumObj", n)

function set_multiobj!(gurobi_model::Gurobi.Model, i::Int, c, priority, weight)
    @assert length(c) == Gurobi.num_vars(gurobi_model)
    oldi = Gurobi.get_int_param(gurobi_model, "ObjNumber")
    Gurobi.set_int_param!(gurobi_model, "ObjNumber", i)
    Gurobi.set_dblattrarray!(gurobi_model, "ObjN", 1, Gurobi.num_vars(gurobi_model), c)
    Gurobi.set_intattr!(gurobi_model, "ObjNPriority", priority)
    Gurobi.set_dblattr!(gurobi_model, "ObjNWeight", weight)
    Gurobi.set_int_param!(gurobi_model, "ObjNumber", oldi)
end

function set_ub!(gurobi_model::Gurobi.Model, u::Vector{Float64})
    @assert length(u) == Gurobi.num_vars(gurobi_model)
    Gurobi.set_dblattrarray!(gurobi_model, "UB", 1, Gurobi.num_vars(gurobi_model), u)
end

function set_rhs!(gurobi_model::Gurobi.Model, rhs::Vector{Float64})
    @assert Gurobi.num_constrs(gurobi_model) == length(rhs)
    Gurobi.set_dblattrarray!(gurobi_model, "RHS", 1, Gurobi.num_constrs(gurobi_model), rhs)
end

function GurobiEnv()
    env = Gurobi.Env()
    # suppress Gurobi logging output
    Gurobi.setparam!(env, "OutputFlag", 0)
    Gurobi.setparam!(env, "Quad", 1)
    Gurobi.setparam!(env, "Method", 4)
    #Gurobi.setparam!(env, "MultiObjMethod", 1)
    Gurobi.setparam!(env, "FeasibilityTol", 1e-9)
    Gurobi.setparam!(env, "OptimalityTol", 1e-9)
    Gurobi.setparam!(env, "MarkowitzTol", 0.9)
    #Gurobi.setparam!(env, "LogToConsole", 0)
    #Gurobi.setparam!(env, "LogFile", path_prefix * "gurobi.log"))
    return env
end

function solve!(gurobi_model::Gurobi.Model)::Bool
    Gurobi.update_model!(gurobi_model)
    Gurobi.optimize(gurobi_model)
    return Gurobi.get_status(gurobi_model) == :optimal
end

function U(V, c, ξ)::Float64
    @argcheck V ≥ 0 && c ≥ 0 && 0 ≤ ξ < Inf
    return c == 0 ? 0. : min(V, c / ξ)
end

function main()
    path_prefix = strip(args["prefix"])

    ξlist = readdlm(path_prefix * strip(args["xi"]))
    @assert all(0 .≤ ξlist .< Inf)

    sto = DataFrames.readtable(path_prefix * strip(args["sto"]); allowcomments=true, separator=' ')
    S = sparse(sto[:i], sto[:k], sto[:s])
    m, n = Base.size(S)
    @assert m < n

    mets = DataFrames.readtable(path_prefix * strip(args["mets"]); allowcomments=true, separator=' ')
    rxns = DataFrames.readtable(path_prefix * strip(args["rxns"]); allowcomments=true, separator=' ')
    @assert Base.size(mets, 1) == m && Base.size(rxns, 1) == n

    @assert all(rxns[:lb] .≤ 0 .≤ rxns[:ub])
    @assert all(rxns[:an] .≥ 0) && all(rxns[:ap] .≥ 0) && all(mets[:bp] .≥ 0) && all(mets[:bn] .≥ 0)
    @assert all(mets[:L] .≤ 0 .≤ mets[:V])
    @assert all(-Inf .< mets[:e] .< Inf) && all(-Inf .< mets[:y] .< Inf)

    c = DataFrames.readtable(path_prefix * strip(args["c"]); allowcomments=true, separator=' ')
    @assert all(c[:mM] .≥ 0)
    @assert all(c[:id] == mets[:id])

    x = zeros(2n + 2m + 2)   # [r+, r-, u+, u-, μ, ϕ]
    upper::Vector{Float64} = [rxns[:ub]; -rxns[:lb]; mets[:V]; -mets[:L]; Inf; 1.]     # upper bounds
    @assert length(upper) == length(x) == 2m + 2n + 2

    # equality constrains
    W = sparse([ S            -S              speye(m)          -speye(m)     -mets[:y]       spzeros(m)
                 rxns[:ap]'    rxns[:an]'     mets[:bp]'        mets[:bn]'    args["A"]       -1 ])

    @assert Base.size(W) == (m + 1, length(x))

    # the problem has the form:
    # max μ s.t. W⋅x = [e,0] and 0 ≤ x ≤ upper.
    # with the secondary objective: min ϕv

    LP = Gurobi.gurobi_model(GurobiEnv(), sense = :maximize,
                             Aeq = W, beq = [collect(mets[:e]); 0],
                             lb = zeros(x), ub = upper, f = zeros(x))

    set_multiobj_n!(LP, 2)
    set_multiobj!(LP, 0, [zeros(2n + 2m); 1;  0], 10, 1.0)  # max μ
    set_multiobj!(LP, 1, [zeros(2n + 2m); 0; -1], 5,  1.0)  # min ϕ

    r = zeros(n)
    u = zeros(m)
    s = zeros(m)

    xout = open(path_prefix * strip(args["out"]) * ".x", "w")
    uout = open(path_prefix * strip(args["out"]) * ".u", "w")
    rout = open(path_prefix * strip(args["out"]) * ".r", "w")
    sout = open(path_prefix * strip(args["out"]) * ".s", "w")

    write(xout, "xi\tmu\tphi\n")
    write(uout, "xi"); for met in mets[:id]; write(uout, "\t" * met); end; write(uout, "\n")
    write(rout, "xi"); for rxn in rxns[:id]; write(rout, "\t" * rxn); end; write(rout, "\n")
    write(sout, "xi"); for met in mets[:id]; write(sout, "\t" * met); end; write(sout, "\n")

    for ξ in ξlist
        # update uptake bounds
        view(upper, 2n + (1:m)) .= U.(mets[:V], c[:mM], ξ)
        set_ub!(LP, upper)

        solve!(LP) || return
        x .= Gurobi.get_solution(LP)
        r .= x[1:n] .- x[n + (1:n)]
        u .= x[2n + (1:m)] .- x[2n + m + (1:m)]
        s .= c[:mM] .- u * ξ
        μ = x[2n + 2m + 1]
        ϕ = x[2n + 2m + 2]

        write(xout, "$ξ\t$μ\t$ϕ\n")
        @printf "ξ=%-7.13f μ=%-7.13f ϕ=%-7.13f\n" ξ μ ϕ

        write(uout, "$ξ"); for i = 1:m; write(uout, "\t$(u[i])"); end; write(uout, "\n")
        write(rout, "$ξ"); for k = 1:n; write(rout, "\t$(r[k])"); end; write(rout, "\n")
        write(sout, "$ξ"); for i = 1:m; write(sout, "\t$(s[i])"); end; write(sout, "\n")
    end

    close(xout)
    close(uout)
    close(rout)
    close(sout)
end

main()


module Netpbm

using FileIO, ImageCore

# Note: there is no endian standard, but netpbm is big-endian
const is_little_endian = ENDIAN_BOM == 0x04030201
const ufixedtype = Dict(10=>N6f10, 12=>N4f12, 14=>N2f14, 16=>N0f16)

function load(f::Union{File{format"PBMBinary"},File{format"PGMBinary"},File{format"PPMBinary"}})
    open(f) do s
        skipmagic(s)
        load(s)
    end
end

function load(f::Union{File{format"PBMText"},File{format"PGMText"},File{format"PPMText"}})
    open(f) do s
        skipmagic(s)
        load(s)
    end
end

function load(s::Stream{format"PBMBinary"})
    io = stream(s)
    w, h = parse_netpbm_size(io)
    dat = BitArray(undef,w, h)
    nbytes_per_row = ceil(Int, w/8)
    for irow = 1:h, j = 1:nbytes_per_row
        tmp = read(io, UInt8)
        offset = (j-1)*8
        for k = 1:min(8, w-offset)
            dat[offset+k, irow] = (tmp>>>(8-k))&0x01
        end
    end
    PermutedDimsArray(dat, (2,1))
end

function load(s::Stream{format"PGMBinary"})
    io = stream(s)
    w, h = parse_netpbm_size(io)
    maxval = parse_netpbm_maxval(io)
    if maxval <= 255
        dat8 = Array{UInt8}(undef, h, w)
        readio!(io, dat8)
        return reinterpret(Gray{N0f8}, dat8)
    elseif maxval <= typemax(UInt16)
        datraw = Array{UInt16}(undef, h, w)
        readio!(io, datraw)
        # Determine the appropriate Normed type
        T = ufixedtype[ceil(Int, log2(maxval)/2)<<1]
        return reinterpret(Gray{T}, datraw)
    else
        error("Image file may be corrupt. Are there really more than 16 bits in this image?")
    end
end

function load(s::Stream{format"PPMBinary"})
    io = stream(s)
    w, h = parse_netpbm_size(io)
    maxval = parse_netpbm_maxval(io)
    local dat
    if maxval <= 255
        dat8 = Array{UInt8}(undef, 3, h, w)
        readio!(io, dat8)
        return reshape(reinterpret(RGB{N0f8}, dat8), (h, w))
    elseif maxval <= typemax(UInt16)
        datraw = Array{UInt16}(undef, 3, h, w)
        readio!(io, datraw)
        # Determine the appropriate Normed type
        T = ufixedtype[ceil(Int, log2(maxval)/2)<<1]
        return reshape(reinterpret(RGB{T}, datraw), (h, w))
    else
        error("Image file may be corrupt. Are there really more than 16 bits in this image?")
    end
end

function load(s::Stream{format"PBMText"})
    io = stream(s)
    w, h = parse_netpbm_size(io)
    dat = BitArray(undef, h, w)
    nbytes_per_row = ceil(Int, w/8)
    readtextio!(io, dat)
    return(dat)
end

function load(s::Stream{format"PGMText"})
    io = stream(s)
    w, h = parse_netpbm_size(io)
    maxval = parse_netpbm_maxval(io)
    if maxval <= 255
        dat8 = Array{UInt8}(undef, h, w)
        readtextio!(io, dat8)
        return reinterpret(Gray{N0f8}, dat8)
    elseif maxval <= typemax(UInt16)
        datraw = Array{UInt16}(undef, h, w)
        readtextio!(io, datraw)
        # Determine the appropriate Normed type
        T = ufixedtype[ceil(Int, log2(maxval)/2)<<1]
        return reinterpret(Gray{T}, datraw)
    else
        error("Image file may be corrupt. Are there really more than 16 bits in this image?")
    end
end

function load(s::Stream{format"PPMText"})
    io = stream(s)
    w, h = parse_netpbm_size(io)
    maxval = parse_netpbm_maxval(io)
    local dat
    if maxval <= 255
        dat8 = Array{UInt8}(undef, 3, h, w)
        readtextio!(io, dat8)
        return reshape(reinterpret(RGB{N0f8}, dat8), (h, w))
    elseif maxval <= typemax(UInt16)
        datraw = Array{UInt16}(undef, 3, h, w)
        readtextio!(io, datraw)
        # Determine the appropriate Normed type
        T = ufixedtype[ceil(Int, log2(maxval)/2)<<1]
        return reshape(reinterpret(RGB{T}, datraw), (h, w))
    else
        error("Image file may be corrupt. Are there really more than 16 bits in this image?")
    end
end

@noinline function readio!(io, dat::AbstractMatrix{T}) where {T}
    axh, axw = axes(dat)
    for i = axh, j = axw  # io is stored in row-major format
        dat[i,j] = default_swap(read(io, T))
    end
    dat
end

@noinline function readio!(io, dat::AbstractArray{T,3}) where {T}
    size(dat, 1) == 3 || throw(DimensionMismatch("must be of size 3 in first dimension, got $(size(dat, 1))"))
    axh, axw = axes(dat, 2), axes(dat, 3)
    for i = axh, j = axw, k = 1:3  # io is stored row-major, color-first
        dat[k,i,j] = default_swap(read(io, T))
    end
    dat
end

@noinline function readtextio!(io, dat::AbstractMatrix{T}) where {T}
    axh, axw = axes(dat)
    for i = axh, j = axw  # io is stored in row-major format
        dat[i,j] = parsenextint(io)
    end
    dat
end

@noinline function readtextio!(io, dat::AbstractArray{T,3}) where {T}
    size(dat, 1) == 3 || throw(DimensionMismatch("must be of size 3 in first dimension, got $(size(dat, 1))"))
    axh, axw = axes(dat, 2), axes(dat, 3)
    for i = axh, j = axw, k = 1:3  # io is stored row-major, color-first
        dat[k,i,j] = parsenextint(io)
    end
    dat
end

function save(filename::File{format"PBMBinary"}, img; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    open(filename, "w") do s
        io = stream(s)
        write(io, "P4\n")
        write(io, "# pbm file written by Julia\n")
        save(s, img, mapf=mapf)
    end
end

function save(filename::File{format"PGMBinary"}, img; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    open(filename, "w") do s
        io = stream(s)
        write(io, "P5\n")
        write(io, "# pgm file written by Julia\n")
        save(s, img, mapf=mapf)
    end
end

function save(filename::File{format"PPMBinary"}, img; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    open(filename, "w") do s
        io = stream(s)
        write(io, "P6\n")
        write(io, "# ppm file written by Julia\n")
        save(s, img, mapf=mapf)
    end
end

function save(filename::File{format"PBMText"}, img; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    open(filename, "w") do s
        io = stream(s)
        write(io, "P1\n")
        write(io, "# pbm file written by Julia\n")
        save(s, img, mapf=mapf)
    end
end

function save(filename::File{format"PGMText"}, img; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    open(filename, "w") do s
        io = stream(s)
        write(io, "P2\n")
        write(io, "# pgm file written by Julia\n")
        save(s, img, mapf=mapf)
    end
end

function save(filename::File{format"PPMText"}, img; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    open(filename, "w") do s
        io = stream(s)
        write(io, "P3\n")
        write(io, "# ppm file written by Julia\n")
        save(s, img, mapf=mapf)
    end
end

function save(s::Stream, img::AbstractMatrix; mapf=identity, mapi=nothing)
    mapf = kwrename(:mapf, mapf, :mapi, mapi, :save)
    save(s, img, mapf)
end

@noinline function save(s::Stream{format"PBMBinary"}, img::AbstractMatrix{T}, mapf) where {T<:Number}
    axh, axw = axes(img)
    w = length(axw)
    write(s, "$(w) $(length(axh))\n")
    dat = PermutedDimsArray(img, (2,1))
    nbytes_per_row = ceil(Int, w/8)
    for irow = axh, j = 1:nbytes_per_row
        tmp = UInt8(0)
        offset = (j-1)*8
        for k = 1:min(8, w-offset)
            tmp |= round(UInt8, mapf(dat[offset+k, irow]) << (8-k))
        end
        write(s, UInt8(tmp))
    end
    nothing
end

@noinline function save(s::Stream{format"PGMBinary"}, img::AbstractMatrix{T}, mapf) where {T<:Union{Gray,Number}}
    axh, axw = axes(img)
    Tout, mx = pnmmax(img)
    if sizeof(Tout) > 2
        error("element type $Tout (from $T) not supported")
    end
    write(s, "$(length(axw)) $(length(axh))\n$mx\n")
    for i = axh, j = axw  # s is stored in row-major format
        write(s, default_swap(round(Tout, mx*gray(mapf(img[i,j])))))
    end
    nothing
end

@noinline function save(s::Stream{format"PPMBinary"}, img::AbstractMatrix{T}, mapf) where {T<:Color}
    axh, axw = axes(img)
    Tout, mx = pnmmax(img)
    if sizeof(Tout) > 2
        error("element type $Tout (from $T) not supported")
    end
    write(s, "$(length(axw)) $(length(axh))\n$mx\n")
    for i = axh, j = axw  # io is stored row-major, color-first
        c = RGB(mapf(img[i,j]))
        write(s, default_swap(round(Tout, mx*red(c))))
        write(s, default_swap(round(Tout, mx*green(c))))
        write(s, default_swap(round(Tout, mx*blue(c))))
    end
    nothing
end

@noinline function save(s::Stream{format"PBMText"}, img::AbstractMatrix{T}, mapf) where {T<:Number}
    axh, axw = axes(img)
    w = length(axw)
    write(s, "$(w) $(length(axh))\n")
    linelen = offset = 0
    maxdigits = 1
    for i = axh, j = axw  # s is stored in row-major format
        if linelen + maxdigits + 1 > 70
            write(s, "\n")
            linelen = 0
        elseif linelen > 0
            write(s, " ")
            linelen += 1
        end
        write(s, Bool(mapf(round(img[i,j]))) ? "1" : "0")
        linelen += maxdigits
        offset += 1
        if offset == w
            write(s, "\n")
            linelen = 0
        end
    end
    nothing
end

@noinline function save(s::Stream{format"PGMText"}, img::AbstractMatrix{T}, mapf) where {T<:Union{Gray,Number}}
    axh, axw = axes(img)
    Tout, mx = pnmmax(img)
    if sizeof(Tout) > 2
        error("element type $Tout (from $T) not supported")
    end
    w = length(axw)
    write(s, "$(w) $(length(axh))\n$mx\n")
    linelen = offset = 0
    maxdigits = round(Int, log10(mx + 1), RoundUp)
    for i = axh, j = axw  # s is stored in row-major format
        value = round(Tout, mx*gray(mapf(img[i,j])))
        if linelen + maxdigits + 1 > 70
            write(s, "\n")
            linelen = 0
        elseif linelen > 0
            write(s, " ")
            linelen += 1
        end
        write(s, lpad("$(Int(value))", maxdigits))
        linelen += maxdigits
        offset += 1
        if offset == w
            write(s, "\n")
            linelen = 0
        end
    end
    nothing
end

@noinline function save(s::Stream{format"PPMText"}, img::AbstractMatrix{T}, mapf) where {T<:Color}
    axh, axw = axes(img)
    Tout, mx = pnmmax(img)
    if sizeof(Tout) > 2
        error("element type $Tout (from $T) not supported")
    end
    w = length(axw)
    write(s, "$(w) $(length(axh))\n$mx\n")
    linelen = offset = 0
    maxdigits = round(Int, log10(mx + 1), RoundUp)
    for i = axh, j = axw  # io is stored row-major, color-first
        c = RGB(mapf(img[i,j]))
        r,g,b = round.(Tout, mx.*(red(c),green(c),blue(c)))
        if linelen + 3*maxdigits + 1 > 70
            write(s, "\n")
            linelen = 0
        elseif linelen > 0
            write(s, "  ")
            linelen += 1
        end
        write(s, string(lpad("$(Int(r))", maxdigits), ' ', lpad("$(Int(g))", maxdigits), ' ', lpad("$(Int(b))", maxdigits)))
        linelen += 2+3*maxdigits
        offset += 1
        if offset == w
            write(s, "\n")
            linelen = 0
        end
    end
    nothing
end

function parse_netpbm_size(stream::IO)
    (parsenextint(stream), parsenextint(stream))
end

function parse_netpbm_maxval(stream::IO)
    parsenextint(stream)
end

function parsenextint(stream::IO)
    # ikirill: ugly, but I can't figure out a better way
    skipchars(isspace, stream, linecomment='#')
    from = position(stream)
    mark(stream)
    skipchars(isdigit, stream)
    to = position(stream)
    reset(stream)
    parse(Int, String(read(stream, to-from+1)))
end

function pnmmax(img::AbstractArray{T}) where {T}
    if isconcretetype(T)
        return pnmmax(eltype(T))
    end
    # Determine the concrete type that can hold all the elements
    S = typeof(first(img))
    for val in img
        S = promote_type(S, typeof(val))
    end
    pnmmax(eltype(S))
end

pnmmax(::Type{T}) where {T<:AbstractFloat} = UInt8, 255
function pnmmax(::Type{U}) where {U<:Normed}
    FixedPointNumbers.rawtype(U), reinterpret(one(U))
end
pnmmax(::Type{T}) where {T<:Unsigned} = T, typemax(T)

mybswap(i::Integer)  = bswap(i)
mybswap(i::Normed)   = bswap(i)
mybswap(c::Colorant) = mapc(bswap, c)
mybswap(c::RGB24) = c

const default_swap = is_little_endian ? mybswap : identity

function kwrename(newname, newval, oldname, oldval, caller::Symbol)
    if oldval !== nothing
        Base.depwarn("keyword $oldname has been renamed $newname", caller)
        return oldval
    end
    newval
end

end # module


tomlpath = joinpath(@__DIR__, "sbm_gwf_config.toml")
config = Wflow.Config(tomlpath)

model = Wflow.initialize_sbm_gwf_model(config)
@unpack network = model

model = Wflow.update_sbm_gwf(model)

# test if the first timestep was written to the CSV file
flush(model.writer.csv_io)  # ensure the buffer is written fully to disk
@testset "CSV output" begin
    csv = CSV.File(model.writer.csv_path)
    row = csv[1]

    @test row.time == DateTime("2000-06-01T00:00:00")
    @test row.Q_av ≈ 0.016260519762890745
    @test row.head ≈ 1.8416896245327632
end

@testset "first timestep" begin
    sbm = model.vertical

    @test model.clock.iteration == 2
    @test sbm.altitude[1] == 1.937000036239624
    @test sbm.θₛ[1] == 0.44999998807907104
    @test sbm.runoff[1] == 0.0
    @test sbm.soilevap[1] == 0.0
    @test sbm.transpiration[1] == 0.6117526566330049
end

# run the second timestep
model = Wflow.update_sbm_gwf(model)

@testset "second timestep" begin
    sbm = model.vertical
    @test sbm.altitude[1] == 1.937000036239624
    @test sbm.θₛ[1] == 0.44999998807907104
    @test sbm.runoff[1] == 0.0
    @test sbm.soilevap[1] == 0.0
    @test sbm.transpiration[1] == 1.0122634204681036
end

@testset "overland flow" begin
    q = model.lateral.land.q_av
    @test sum(q) ≈ 0.0
end

@testset "river flow" begin
    q = model.lateral.river.q_av
    @test sum(q) ≈ 0.03515256649675796
    @test q[6] ≈ 0.00795266793733648
    @test q[13] ≈  0.0005980549864213564
    @test q[network.river.order[end]] ≈  0.008482646477950898
end

@testset "groundwater" begin
    gw = model.lateral.subsurface
    @test gw.river.stage[1] ≈ 1.21057153442702
    @test gw.flow.aquifer.head[19] ≈ 1.7999999523162842
    @test gw.river.flux[1] ≈ -50.751898489778426
    @test gw.drain.flux[1] ≈  0.0
    @test gw.recharge.rate[19] ≈  -0.0014241196552847502
end

Wflow.close_files(model, delete_output = false)


module ComplexBC

import JuMP
import MathOptInterface
import BranchAndBound
import PowerModels
import Mosek, MosekTools
import Ipopt
import LinearAlgebra: norm, rank

const MOI = MathOptInterface
const BB = BranchAndBound
const PM = PowerModels

const MOSEK_OPTIMIZER = JuMP.optimizer_with_attributes(Mosek.Optimizer, "QUIET" => true)
const MAX_NODE_ALLOWED = 10000

include("utils.jl")
include("data_structures.jl")
include("pm_exts.jl")
include("key_steps.jl")
include("bb_exts.jl")
include("initialize.jl")

function main(file::String)::Nothing
    data = PM.parse_file(file)
    pm = PM.instantiate_model(data, NodeWRMPowerModel, PM.build_opf)
    optimizer = JuMP.optimizer_with_attributes(Mosek.Optimizer, "QUIET" => true)
    # optimizer = optimizer_with_attributes(SCS.Optimizer, "max_iters" => 10000, "verbose" => 0)
    JuMP.set_optimizer(pm.model, optimizer)

    tree, node = initialize(pm, file)

    @time BB.run(tree)

    if isempty(node.auxiliary_data["rank1"])
        println("No rank-1 solution found")
    else
        (bound, id) = findmin(node.auxiliary_data["rank1"])
        println("Best bound obtained at $(id), bound value $(bound)")
    end
end

end # module


let
    mpsim = MPsim()

    attrib = Attribute("Attr A")
    addPossibleAttributeValue!( attrib, "Val AA", "Val AB" )
    addSimulationAttribute!( mpsim, attrib )

    node = BaseNode("Test A")
    addNodeRequirement!( node, ("Attr A", "Val AA") )
    addSimulationBaseNode!( mpsim, node )

    recr = Recruitment("EW")
    setRecruitmentTarget!( recr, "Test A" )
    setRecruitmentSchedule!( recr, 12 )
    setRecruitmentFixed!( recr, 10 )
    setRecruitmentAgeFixed!( recr, 18*12 )
    addSimulationRecruitment!( mpsim, recr )

    retir = Retirement()
    setRetirementCareerLength!( retir, 60 )
    setRetirementSchedule!( retir, 12 )
    setSimulationRetirement!( mpsim, retir )

    setSimulationLength!( mpsim, 120 )
    run( mpsim )
end


## Categorical Scales

mutable struct Cycler{K, V}
    keys::Vector{K}
    values::Vector{V}
    defaults::Vector{Any}
    idx::Int
end

function Cycler(p)
    defaults = vec(collect(Any, p))
    pairs = splice!(defaults, findall(val -> val isa Pair, defaults))
    return Cycler(map(first, pairs), map(last, pairs), defaults, 0)
end

function (c::Cycler)(u)
    i = findfirst(isequal(u), c.keys)
    return if isnothing(i)
        l = length(c.defaults)
        l == 0 && throw(ArgumentError("Key $(repr(u)) not found and no default values are present"))
        c.defaults[mod1(c.idx += 1, l)]
    else
        c.values[i]
    end
end

# Use `Iterators.map` as `map` does not guarantee order
apply_palette(p::Union{AbstractArray, AbstractColorList}, uv) = collect(Iterators.map(Cycler(p), uv))
apply_palette(::Automatic, uv) = eachindex(uv)
apply_palette(p, uv) = map(p, uv)

# TODO: add more customizations?
struct Wrap end

const wrap = Wrap()

function apply_palette(::Wrap, uv)
    ncols = ceil(Int, sqrt(length(uv)))
    return [fldmod1(idx, ncols) for idx in eachindex(uv)]
end

struct CategoricalScale{S, T, U}
    data::S
    plot::T
    palette::U
    label::Union{AbstractString, Nothing}
end

function CategoricalScale(data, palette, label::Union{AbstractString, Nothing})
    return CategoricalScale(data, nothing, palette, label)
end

# Final processing step of a categorical scale
function fitscale(c::CategoricalScale)
    data = c.data
    palette = c.palette
    plot = apply_palette(c.palette, c.data)
    return CategoricalScale(data, plot, palette, c.label)
end

datavalues(c::CategoricalScale) = c.data
plotvalues(c::CategoricalScale) = c.plot
getlabel(c::CategoricalScale) = something(c.label, "")

## Continuous Scales

struct ContinuousScale{T}
    extrema::NTuple{2, T}
    label::Union{AbstractString, Nothing}
end

getlabel(c::ContinuousScale) = something(c.label, "")

# recentering hack to avoid Float32 conversion errors on recent dates
# TODO: remove once Makie supports dates
datetime2float(x::Union{DateTime, Date}) = datetime2float(DateTime(x) - DateTime(2020, 01, 01))
datetime2float(x::Time) = datetime2float(x - Time(0))
datetime2float(x::Period) = Millisecond(x) / Millisecond(1)

"""
    datetimeticks(datetimes::AbstractVector{<:TimeType}, labels::AbstractVector{<:AbstractString})

Generate ticks matching `datetimes` to the corresponding `labels`.
The result can be passed to `xticks`, `yticks`, or `zticks`.
"""
function datetimeticks(datetimes::AbstractVector{<:TimeType}, labels::AbstractVector{<:AbstractString})
    return map(datetime2float, datetimes), labels
end

"""
    datetimeticks(f, datetimes::AbstractVector{<:TimeType})

Compute ticks for the given `datetimes` using a formatting function `f`.
The result can be passed to `xticks`, `yticks`, or `zticks`.
"""
function datetimeticks(f, datetimes::AbstractVector{<:TimeType})
    return datetimeticks(datetimes, map(string∘f, datetimes))
end

# Rescaling methods that do not depend on context
elementwise_rescale(value::Union{TimeType, Period}) = datetime2float(value)
elementwise_rescale(value::Verbatim) = value[]
elementwise_rescale(value) = value

contextfree_rescale(values) = map(elementwise_rescale, values)

rescale(values, ::Nothing) = values

function rescale(values, c::CategoricalScale)
    # Do not rescale continuous data with categorical scale
    scientific_eltype(values) === categorical || return values
    idxs = indexin(values, datavalues(c))
    return plotvalues(c)[idxs]
end

Base.length(c::CategoricalScale) = length(datavalues(c))

function mergelabels(label1, label2)
    return if isnothing(label1)
        nothing
    elseif isequal(label1, label2)
        label1
    elseif isempty(label1)
        label2
    elseif isempty(label2)
        label1
    else
        nothing # no reasonable label found
    end
end

function mergescales(c1::CategoricalScale, c2::CategoricalScale)
    data = mergesorted(c1.data, c2.data)
    palette = assert_equal(c1.palette, c2.palette)
    label = mergelabels(c1.label, c2.label)
    return CategoricalScale(data, palette, label)
end

function mergescales(c1::ContinuousScale, c2::ContinuousScale)
    extrema = extend_extrema(c1.extrema, c2.extrema)
    label = mergelabels(c1.label, c2.label)
    return ContinuousScale(extrema, label)
end

# Logic to create ticks from a scale
# Should take current tick to incorporate information
function ticks(scale::CategoricalScale)
    u = map(to_string, datavalues(scale))
    return (axes(u, 1), u)
end

ticks(scale::ContinuousScale) = ticks(scale.extrema)

ticks((min, max)::NTuple{2, Any}) = automatic

temporal_resolutions(::Type{Date}) = (Year, Month, Day)
temporal_resolutions(::Type{Time}) = (Hour, Minute, Second, Millisecond)
temporal_resolutions(::Type{DateTime}) = (temporal_resolutions(Date)..., temporal_resolutions(Time)...)

function optimal_datetime_range((x_min, x_max)::NTuple{2, T}; k_min=2, k_max=5) where {T<:TimeType}
    local P, start, stop
    for outer P in temporal_resolutions(T)
        start, stop = trunc(x_min, P), trunc(x_max, P)
        (start == x_min) || (start += P(1))
        n = length(start:P(1):stop)
        n ≥ k_min && return start:P(fld1(n, k_max)):stop
    end
    return start:P(1):stop
end

function format_datetimes(datetimes::AbstractVector{DateTime})
    dates, times = Date.(datetimes), Time.(datetimes)
    (dates == datetimes) && return string.(dates)
    isequal(extrema(dates)...) && return string.(times)
    return string.(datetimes)
end

format_datetimes(datetimes::AbstractVector) = string.(datetimes)

function ticks(limits::NTuple{2, TimeType})
    datetimes = optimal_datetime_range(limits)
    return datetime2float.(datetimes), format_datetimes(datetimes)
end

@enum ScientificType categorical continuous geometrical

"""
    scientific_type(T::Type)

Determine whether `T` represents a continuous, geometrical, or categorical variable.
"""
function scientific_type(::Type{T}) where T
    T <: Bool && return categorical
    T <: Union{Number, TimeType} && return continuous
    T <: Verbatim && return geometrical
    T <: Union{Makie.StaticVector, Point, AbstractGeometry} && return geometrical
    T <: AbstractArray && eltype(T) <: Union{Point, AbstractGeometry} && return geometrical
    return categorical
end

"""
    scientific_eltype(v)

Determine whether `v` should be treated as a continuous, geometrical, or categorical array.
"""
scientific_eltype(v::AbstractArray) = scientific_type(eltype(v))

# TODO: Needed for pregrouped data, but ideally should be removed.
scientific_eltype(::Any) = categorical

iscategoricalcontainer(u) = any(el -> scientific_eltype(el) === categorical, u)
iscontinuous(u) = scientific_eltype(u) === continuous

extend_extrema((l1, u1), (l2, u2)) = min(l1, l2), max(u1, u2)

function extrema_finite(v::AbstractArray)
    iter = Iterators.filter(isfinite, skipmissing(v))
    init = typemax(eltype(iter)), typemin(eltype(iter))
    return mapreduce(t -> (t, t), extend_extrema, iter; init)
end

nested_extrema_finite(iter) = mapreduce(extrema_finite, extend_extrema, iter)

push_different!(v, val) = !isempty(v) && isequal(last(v), val) || push!(v, val)

function mergesorted(v1, v2)
    issorted(v1) && issorted(v2) || throw(ArgumentError("Arguments must be sorted"))
    T = promote_type(eltype(v1), eltype(v2))
    v = sizehint!(T[], length(v1) + length(v2))
    i1, i2 = 1, 1
    while i2 ≤ length(v2)
        while i1 ≤ length(v1) && isless(v1[i1], v2[i2])
            push_different!(v, v1[i1])
            i1 += 1
        end
        push_different!(v, v2[i2])
        i2 += 1
    end
    while i1 ≤ length(v1)
        push_different!(v, v1[i1])
        i1 += 1
    end
    return v
end


module taxis
using sequenceCompare
using DataFrames
using JSON
using Iterators

export Point, SequenceRef, LoadData, GetUniqueCoords, GetTableOrderedSubset, ConstructCoordsDatabase, ConstructCoordsDatabase2, PlotPaths, findClosestTrainingExample, findClosestTrainingExampleForTestSet

    immutable Point{T}
      x::T
      y::T
    end

    type SequenceRef
        TripId::Int64
        SequenceIndex::Int64
        LengthRemaining::Int64
    end

    function LoadData(train_path, test_path)
        println("Begin")

        println("loading csv files")
        taxi_df = readtable(train_path)
        taxi_validation_df = readtable(test_path)

        println("loading coords")
        taxi_df[:COORDS] = [float(hcat(JSON.parse(x)...)) for x in  taxi_df[:POLYLINE]]
        taxi_validation_df[:COORDS] = [float(hcat(JSON.parse(x)...)) for x in taxi_validation_df[:POLYLINE]]

        println("getting coords counts")
        taxi_df[:NUM_COORDS] = [int(length(x)/2)::Int64 for x in taxi_df[:COORDS]]
        taxi_validation_df[:NUM_COORDS] = [int(length(x)/2)::Int64 for x in taxi_validation_df[:COORDS]]

        println("deleting unneeded data rows/columns")
        delete!(taxi_validation_df, :POLYLINE)
        delete!(taxi_df, :POLYLINE)

        taxi_df[:START] = [x[:,1] for x in taxi_df[:COORDS]]
        taxi_validation_df[:START] = [x[:,1] for x in taxi_validation_df[:COORDS]]

        taxi_df[:END] = [x[:,end] for x in taxi_df[:COORDS]]
        taxi_validation_df[:END] = [x[:,end] for x in taxi_validation_df[:COORDS]]

        #These examples are not going to be useful!
        deleterows!(taxi_df, find(taxi_df[:NUM_COORDS] .== 0))

    taxi_df[:COORDS_TEST] = [c[:,1:int(floor((0.3 + 0.7*rand(1)[1])*size(c,2)))] for c in taxi_df[:COORDS]]

        println("done!")

        return taxi_df, taxi_validation_df
    end

    function GetUniqueCoords(c, round_len=4)
        c = round(c, round_len)
        c2 = Set([(c[i,1]::Float64, c[i,2]::Float64) for i in 1:size(c, 1)])
        println("all coords: ", length(c), ", unique coords: ", length(c2))
        return c2
    end

    function GetTableOrderedSubset(df, num_records=100000)
        idxs = sort(shuffle([1:size(df,1)])[1:num_records])
        return df[idxs,:]
    end

    # Constructing a coords database

    function ConstructCoordsDatabase(df, round_len=5)
        print(length(df[:TRIP_ID]), ", ", length(df[:COORDS]))
        return ConstructCoordsDatabase2(df[:TRIP_ID], df[:COORDS], round_len)
    end

    function ConstructCoordsDatabase2(TripIds, Coords, round_len=5)
        #println("num trip_ids/coords: ", length(TripIds), " ", length(Coords))
        #print("coords:", Coords)
        D = Dict{(Float64,Float64),Array{SequenceRef,1}}()
        num_coords = size(TripIds,1)
        for i=1:num_coords
            trip_id = TripIds[i]

            coords = Coords[i]
            if length(coords) == 0
                continue
            end

            coords = round(Coords[i],round_len)

            num_coords = size(coords,2)
            #println("num_coords: ", num_coords)
            for j=1:num_coords
                coord_pair = (coords[1,j], coords[2,j])
                #println("coord_pair: ", coord_pair, ", j/num_coords: ", j, "/", num_coords, ", thing: ", [SequenceRef(trip_id, j, num_coords-j)])
                if !haskey(D, coord_pair)
                    D[coord_pair] = [SequenceRef(trip_id, j, num_coords-j)]
                else
                    push!(D[coord_pair], SequenceRef(trip_id, j, num_coords-j))
                end
            end
        end

        return D
    end

    # Finding similar paths using Dynamic Path Warping

    function findClosestTrainingExample(all_train_coords, test_path, w=1)
        num_paths = length(all_train_coords)
        best_dist = 9999.0
        best_path = all_train_coords[1]
        for k=1:num_paths
            train_path = all_train_coords[k]

            if size(train_path, 2) < size(test_path, 2)
                continue
            end

            if w == 1
                dist = sequenceCompare.DTWDistance(train_path, test_path)
            else
                dist = sequenceCompare.DTWDistance(train_path, test_path, w)
            end
            #println("k=", k, " dist=", dist)
            if dist < best_dist
                #println(k, ", old best: ", best_dist, " new best: ", dist)
                best_dist = dist
                best_path = all_train_coords[k]
            end
        end

        return best_path
    end

    function findClosestTrainingExampleForTestSet(all_train_paths, all_test_paths, w=1)
        num_train_paths = length(all_train_paths)
        num_test_paths = length(all_test_paths)
        closest_examples = cell(num_test_paths)
        for k=1:num_test_paths
            if k % 20 == 0
                println(k, "/", num_test_paths, " for ", num_train_paths, " train path examples")
            end
            test_path = all_test_paths[k]
            closest_training_example = findClosestTrainingExample(all_train_paths, test_path, w)
            closest_examples[k] = closest_training_example
        end

        return closest_examples
    end

    function pFindClosestTrainingExampleForTestSet(all_train_paths, all_test_paths)
        getClosestExample = p -> findClosestTrainingExample(all_train_paths, p)
        return pmap(getClosestExample, all_test_paths)
    end

    # Didn't want to throw these away..
    function GetAverageDistanceFromCoordsSequence(coords)
        #Outline of the basic algorithm
        #for coord_pair c_1..c_n
        # estimate <- for i=1:n average(15*(i-1) + average(for c_j in D[c+i] * exp(-|L_j-L_i|) * L_j)
        coords = round(Coords[i],round_len)
    end

    function GetAverageDistancesByCoordsDict(TripIds, Coords, round_len=5)
        distaces = Array{Float64,0}()
        num_coords = size(TripIds,1)
        for i=1:num_coords
            trip_id = TripIds[i]

            coords = Coords[i]
            if length(coords) == 0
                continue
            end

            coords = round(Coords[i],round_len)

            num_coords = size(coords,2)
            #println("num_coords: ", num_coords)
            for j=1:num_coords
                coord_pair = (coords[1,j], coords[2,j])
                #println("coord_pair: ", coord_pair, ", j/num_coords: ", j, "/", num_coords, ", thing: ", [SequenceRef(trip_id, j, num_coords-j)])
                if !haskey(D, coord_pair)
                    D[coord_pair] = [SequenceRef(trip_id, j, num_coords-j)]
                else
                    push!(D[coord_pair], SequenceRef(trip_id, j, num_coords-j))
                end
            end
        end

        return D
    end

# ******************** Visualizations ******************************

    function PlotPaths(coords, show_limit=40)
        num_paths = min(length(coords), show_limit)
        num_coords = length(coords)
        coordsToPlot = coords[randperm(num_coords)[1:num_paths]]
        layers = [layer(x=round(c'[:,1],3),y=round(c'[:,2],3),Geom.point, Geom.path, Theme(default_color=RGB(rand(3)...))) for c in coordsToPlot]
        Gadfly.plot(layers...)
    end

end


# R: number of regions
# G: number of generators (either thermal / VG / storage)
# T: number of operations timesteps per period
# P: number of operations periods (representative)

mutable struct InvestmentProblem{R,G1,G2,G3,I,T,P} <: AbstractProblem{R,G1,G2,G3,I,T,P}

    model::Model
    regionnames::Vector{String}
    periodnames::Vector{String}
    technologies::Technologies{G1,G2,G3,I,R}
    initialconditions::InitialConditions{R,G1,G2,G3}
    discountrate::Float64
    rootscenario::Scenario{R,G1,G2,G3,I,T,P}

    function InvestmentProblem{T}(
        model::Model,
        regionnames::Vector{String},
        periodnames::Vector{String},
        techs::Technologies{G1,G2,G3,I,R},
        initconds::InitialConditions{R,G1,G2,G3},
        discountrate::Float64
    ) where {R,G1,G2,G3,I,T}

        @assert R == length(regionnames)
        P = length(periodnames)

        new{R,G1,G2,G3,I,T,P}(model, regionnames, periodnames,
                              techs, initconds, discountrate)

    end

end

function InvestmentProblem(
    regionnames::Vector{String},
    periodnames::Vector{String},
    techs::Technologies{G1,G2,G3,I,R},
    initconds::InitialConditions{R,G1,G2,G3},
    discountrate::Float64,
    rootname::String,
    investments::Investments{R,G1,G2,G3},
    operations::Operations{R,G1,G2,G3,I,T,P},
    markets::Markets{R,T,P},
    periodweights::Vector{Float64},
    optimizer::Optimizer
) where {R,G1,G2,G3,I,T,P}

    invprob = InvestmentProblem{T}(
        Model(optimizer), regionnames, periodnames,
        techs, initconds, discountrate)

    root = Scenario(invprob, rootname,
                    investments, operations, markets, periodweights)
    invprob.rootscenario = root

    return invprob

end

# For now, loading in single-scenario only
function InvestmentProblem(
    folder::String, optimizer::Optimizer
)

    # Load representative periods
    perioddata = DataFrame!(CSV.File(joinpath(folder, "periods.csv"),
                                     types=period_types))
    n_periods = length(perioddata.name)
    periodlookup = Dict(zip(perioddata.name, 1:n_periods))

    technicalpath = joinpath(folder, "technical")

    n_timesteps = 24 # TODO: Load these from a file
    discountrate = 0.95

    # Load regions

    regiondata = DataFrame!(CSV.File(joinpath(technicalpath, "regions.csv"),
                                     type=String))
    n_regions = length(regiondata.name)
    regionlookup = Dict(zip(regiondata.name, 1:n_regions))

    # Load technologies

    thermaldata = DataFrame!(CSV.File(joinpath(technicalpath, "thermal.csv")))
    thermaltechs = ThermalGenerators(
        thermaldata.name, thermaldata.owner,
        thermaldata.mingen, thermaldata.maxgen,
        thermaldata.minuptime, thermaldata.mindowntime,
        thermaldata.maxrampup, thermaldata.maxrampdown,
        thermaldata.capacitycredit)
    n_thermals = length(thermaltechs.name)
    thermallookup = Dict(zip(thermaldata.name, 1:n_thermals))

    variabledata = DataFrame!(CSV.File(joinpath(technicalpath, "variable.csv")))
    variabletechs = VariableGenerators(
        variabledata.name, variabledata.owner, variabledata.maxgen,
        variabledata.capacitycredit)
    n_variables = length(variabletechs.name)
    variablelookup = Dict(zip(variabledata.name, 1:n_variables))

    storagedata = DataFrame!(CSV.File(joinpath(technicalpath, "storage.csv")))
    storagetechs = StorageDevices(
        storagedata.name, storagedata.owner,
        storagedata.maxcharge, storagedata.maxdischarge, storagedata.maxenergy,
        storagedata.chargeefficiency, storagedata.dischargeefficiency,
        storagedata.carryoverefficiency, storagedata.capacitycredit)
    n_storages = length(storagetechs.name)
    storagelookup = Dict(zip(storagedata.name, 1:n_storages))

    interfacedata = DataFrame!(CSV.File(
        joinpath(technicalpath, "interface.csv"), type=String))

    from_idx = getindex.(Ref(regionlookup), interfacedata.from)
    to_idx = getindex.(Ref(regionlookup), interfacedata.to)
    interfaces = Interfaces{n_regions}(
        interfacedata.name, tuple.(from_idx, to_idx))
    n_interfaces = length(interfacedata.name)

    techs = Technologies(thermaltechs, variabletechs, storagetechs, interfaces)

    # Load starting builds / options

    initialdata =
        DataFrame!(CSV.File(joinpath(folder, "initialconditions.csv"),
                            types=initialcondition_types))

    thermalstarts = InitialInvestments(
        zeros(Int, n_regions, n_thermals), zeros(Int, n_regions, n_thermals))

    variablestarts = InitialInvestments(
        zeros(Int, n_regions, n_variables), zeros(Int, n_regions, n_variables))

    storagestarts = InitialInvestments(
        zeros(Int, n_regions, n_storages), zeros(Int, n_regions, n_storages))

    for r in eachrow(initialdata)

        region_idx = regionlookup[r.region]

        thermal_idx = get(thermallookup, r.class, nothing)
        if !isnothing(thermal_idx)
            thermalstarts.options[region_idx, thermal_idx] = r.optioned
            thermalstarts.builds[region_idx, thermal_idx] = r.built
            continue
        end

        variable_idx = get(variablelookup, r.class, nothing)
        if !isnothing(variable_idx)
            variablestarts.options[region_idx, variable_idx] = r.optioned
            variablestarts.builds[region_idx, variable_idx] = r.built
            continue
        end

        storage_idx = get(storagelookup, r.class, nothing)
        if !isnothing(storage_idx)
            storagestarts.options[region_idx, storage_idx] = r.optioned
            storagestarts.builds[region_idx, storage_idx] = r.built
            continue
        end

        error("Unknown class $(r.class) listed in initial conditions")

    end

    initconds = InitialConditions(thermalstarts, variablestarts, storagestarts)

    # Load scenario data

    scenariospath = joinpath(folder, "scenarios")
    scenariosdata = DataFrame!(CSV.File(joinpath(scenariospath, "tree.csv"),
                                        types=scenarios_tree_types))

    root_idx = findfirst(ismissing, scenariosdata.parent)
    rootname = scenariosdata.name[root_idx]
    rootpath = joinpath(scenariospath, rootname)
    investments, operations, markets =
        loadscenario(regionlookup, techs, periodlookup, n_timesteps, rootpath)

    result = InvestmentProblem(
        regiondata.name, perioddata.name, techs, initconds, discountrate,
        rootname, investments, operations, markets,
        perioddata.weight, optimizer)

    function makechildren!(parent::Scenario)

        for (childname, prob) in getchildren(parent.name, scenariosdata)

            investments, operations, markets = loadscenario(
                regionlookup, techs, periodlookup, n_timesteps,
                joinpath(scenariospath, childname))

            child = Scenario(parent, childname, prob,
                             investments, operations, markets,
                             perioddata.weight)

            makechildren!(child)

        end

    end

    makechildren!(result.rootscenario)

    return result

end

function solve!(invprob::InvestmentProblem; debug::Bool=false)
    @objective(invprob.model, Max, welfare(invprob.rootscenario))
    debug && open(f -> println(f, invprob.model), "problem.txt", "w")
    optimize!(invprob.model)
    return
end

getchildren(scenario::String, data::DataFrame) = [(row.name, row.probability)
    for row in eachrow(data) if !ismissing(row.parent) && row.parent == scenario]

scenarios(invprob::InvestmentProblem) = scenarios(invprob.rootscenario)

function scenarios(root::Scenario)
    iszero(length(root.children)) && return [(root, true)]
    return vcat((root, false), scenarios.(root.children)...)
end


#=
    Loads modelling
 =#

mutable struct Demand <: AbstractDemand
     # Variables
     carrier::EnergyCarrier
     timestamp::Array{DateTime,3}

     # Inner constructor
     Demand(; carrier = Electricity()) = new(carrier)
end

### Preallocation
function preallocate!(ld::Demand, nh::Int64, ny::Int64, ns::Int64)
    ld.carrier.power = convert(SharedArray,zeros(nh, ny, ns))
    ld.timestamp = Array{DateTime}(undef,(nh, ny, ns))
    return ld
end


export MelodyOneHotEncoding
import ..encode_event, ..decode_event

using ..NoteSequences: MIN_MIDI_PITCH, MAX_MIDI_PITCH

"""
    MelodyOneHotEncoding <: Any

One-Hot encoder for `Melody` events.

## Fields
* `minpitch::Int`: Minimum melody pitch to allow for encoding.
* `maxpitch::Int`: Maximum melody pitch to allow for encoding.
* `num_classes::Int`: Total number of event classes.
"""
struct MelodyOneHotEncoding
    minpitch::Int
    maxpitch::Int
    num_classes::Int

    function MelodyOneHotEncoding(minpitch::Int, maxpitch::Int)
        maxpitch <= minpitch &&
            throw(error("max pitch should be greater than min pitch"))
        minpitch < MIN_MIDI_PITCH &&
            throw(error("min pitch is lesser than 0 (minimum allowed midi pitch)"))
        maxpitch > MAX_MIDI_PITCH + 1 &&
            throw(error("max pitch is greater than 128"))

        num_classes = maxpitch - minpitch + NUM_SPECIAL_MELODY_EVENTS
        new(minpitch, maxpitch, num_classes)
    end
end

function Base.getproperty(encoder::MelodyOneHotEncoding, sym::Symbol)
    if sym === :labels
        return 1:encoder.num_classes
    elseif sym === :defaultevent
        return MELODY_NO_EVENT
    else
        getfield(encoder, sym)
    end
end

"""
    encode_event(event::Int, encoder::MelodyOneHotEncoding)

Encodes a `Melody` event to its corresponding one hot index.
"""
function encode_event(event::Int, encoder::MelodyOneHotEncoding)
    0 <= event < encoder.minpitch &&
        throw(error("Melody event is less than min melody pitch"))
    event >= encoder.maxpitch &&
        throw(error("Melody event is greater than max melody pitch"))
    event < -NUM_SPECIAL_MELODY_EVENTS &&
        throw(error("Invalid melody event"))

    if event < 0
        index = event + NUM_SPECIAL_MELODY_EVENTS
    else
        index = event - encoder.minpitch + NUM_SPECIAL_MELODY_EVENTS
    end

    # Add one to convert to 1-based indexing (starts at 1, ends at num_classes)
    index + 1
end

"""
    decode_event(index::Int, encoder::MelodyOneHotEncoding)

Decodes a one hot index to its corresponding `Melody` event.
"""
function decode_event(index::Int, encoder::MelodyOneHotEncoding)
    if index < NUM_SPECIAL_MELODY_EVENTS + 1
        event = index - NUM_SPECIAL_MELODY_EVENTS
    else
        event = index - NUM_SPECIAL_MELODY_EVENTS + encoder.minpitch
    end

    # Subtract one to convert from 1-based indexing
    event - 1
end