5  Mitigation Strategies

models = [
    :LogisticRegression, 
    :FluxModel, 
    :FluxEnsemble,
]
opt = Flux.Descent(0.01) 
generators = Dict(
    :Generic=>GenericGenerator(opt = opt, decision_threshold=0.5),
    :Latent=>REVISEGenerator(opt = opt),
    :Generic_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
    :Gravitational=>GravitationalGenerator(opt = opt),
    :ClapROAR=>ClapROARGenerator(opt = opt)
)

5.1 Synthetic

max_obs = 1000
catalogue = load_synthetic(max_obs)
choices = [
    :linearly_separable, 
    :overlapping, 
    :circles, 
    :moons,
]
data_sets = filter(p -> p[1] in choices, catalogue)

experiments = set_up_experiments(data_sets,models,generators)

n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
T = 100
using Serialization
results = run_experiments(
    experiments;
    save_path=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T
)
Serialization.serialize(joinpath(output_path,"results_synthetic.jls"),results)

5.2 Plots

using Serialization
results = Serialization.deserialize(joinpath(output_path,"results_synthetic.jls"))

using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
    plt = plot(res)
    Images.save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
    line_charts[data_name] = plt
    plt = plot(res,maximum(res.output.n))
    Images.save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
    errorbar_charts[data_name] = plt
end

5.2.1 Line Charts

Figure 5.1 shows the evolution of the evaluation metrics over the course of the experiment.

choices = [
    :linearly_separable, 
    :overlapping, 
    :circles, 
    :moons,
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart") .&& .!contains.(readdir(www_artifact_path),"latent")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
    display(load(img))
end

(a) California Housing

(b) Circles

(c) Credit Default

(d) GMSC

Figure 5.1: Line Charts

5.2.2 Error Bar Charts

Figure 5.2 shows the evaluation metrics at the end of the experiments.

choices = [
    :linearly_separable, 
    :overlapping, 
    :circles, 
    :moons,
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart") .&& .!contains.(readdir(www_artifact_path),"latent")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
    display(load(img))
end

(a) California Housing

(b) Circles

(c) Credit Default

(d) GMSC

Figure 5.2: Error Bar Charts

5.3 Bootstrap

n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_synthetic.csv"))

┌──────────┬─────────┬────────────────────┬────────────────────┬──────────────────────┬──────────────┐
│     name │   scope │               data │              model │            generator │ p_value_mean │
│ String31 │ String7 │           String31 │           String31 │             String31 │      Float64 │
├──────────┼─────────┼────────────────────┼────────────────────┼──────────────────────┼──────────────┤
│      mmd │  domain │        overlapping │ LogisticRegression │              Generic │          0.0 │
│      mmd │  domain │        overlapping │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │  domain │        overlapping │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │  domain │        overlapping │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │  domain │        overlapping │ LogisticRegression │               Latent │          0.0 │
│      mmd │  domain │        overlapping │          FluxModel │              Generic │          0.0 │
│      mmd │  domain │        overlapping │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │  domain │        overlapping │          FluxModel │             ClapROAR │          0.0 │
│      mmd │  domain │        overlapping │          FluxModel │        Gravitational │          0.0 │
│      mmd │  domain │        overlapping │          FluxModel │               Latent │          0.0 │
│      mmd │  domain │        overlapping │       FluxEnsemble │              Generic │          0.0 │
│      mmd │  domain │        overlapping │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │  domain │        overlapping │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │  domain │        overlapping │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │  domain │        overlapping │       FluxEnsemble │               Latent │          0.0 │
│      mmd │  domain │ linearly_separable │ LogisticRegression │              Generic │          0.0 │
│      mmd │  domain │ linearly_separable │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │  domain │ linearly_separable │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │  domain │ linearly_separable │ LogisticRegression │        Gravitational │        0.092 │
│      mmd │  domain │ linearly_separable │ LogisticRegression │               Latent │         0.75 │
│      mmd │  domain │ linearly_separable │          FluxModel │              Generic │          0.0 │
│      mmd │  domain │ linearly_separable │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │  domain │ linearly_separable │          FluxModel │             ClapROAR │          0.0 │
│      mmd │  domain │ linearly_separable │          FluxModel │        Gravitational │        0.078 │
│      mmd │  domain │ linearly_separable │          FluxModel │               Latent │        0.742 │
│      mmd │  domain │ linearly_separable │       FluxEnsemble │              Generic │          0.0 │
│      mmd │  domain │ linearly_separable │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │  domain │ linearly_separable │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │  domain │ linearly_separable │       FluxEnsemble │        Gravitational │         0.05 │
│      mmd │  domain │ linearly_separable │       FluxEnsemble │               Latent │        0.724 │
│      mmd │  domain │            circles │ LogisticRegression │              Generic │          1.0 │
│      mmd │  domain │            circles │ LogisticRegression │ Generic_conservative │          1.0 │
│      mmd │  domain │            circles │ LogisticRegression │             ClapROAR │          1.0 │
│      mmd │  domain │            circles │ LogisticRegression │        Gravitational │          1.0 │
│      mmd │  domain │            circles │ LogisticRegression │               Latent │          1.0 │
│      mmd │  domain │            circles │          FluxModel │              Generic │        0.994 │
│      mmd │  domain │            circles │          FluxModel │ Generic_conservative │        0.992 │
│      mmd │  domain │            circles │          FluxModel │             ClapROAR │        0.992 │
│      mmd │  domain │            circles │          FluxModel │        Gravitational │        0.998 │
│      mmd │  domain │            circles │          FluxModel │               Latent │          1.0 │
│      mmd │  domain │            circles │       FluxEnsemble │              Generic │         0.99 │
│      mmd │  domain │            circles │       FluxEnsemble │ Generic_conservative │        0.996 │
│      mmd │  domain │            circles │       FluxEnsemble │             ClapROAR │        0.984 │
│      mmd │  domain │            circles │       FluxEnsemble │        Gravitational │        0.998 │
│      mmd │  domain │            circles │       FluxEnsemble │               Latent │          1.0 │
│      mmd │  domain │              moons │ LogisticRegression │              Generic │          0.0 │
│      mmd │  domain │              moons │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │  domain │              moons │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │  domain │              moons │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │  domain │              moons │ LogisticRegression │               Latent │          0.0 │
│      mmd │  domain │              moons │          FluxModel │              Generic │          0.0 │
│      mmd │  domain │              moons │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │  domain │              moons │          FluxModel │             ClapROAR │          0.0 │
│      mmd │  domain │              moons │          FluxModel │        Gravitational │          0.0 │
│      mmd │  domain │              moons │          FluxModel │               Latent │          0.0 │
│      mmd │  domain │              moons │       FluxEnsemble │              Generic │          0.0 │
│      mmd │  domain │              moons │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │  domain │              moons │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │  domain │              moons │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │  domain │              moons │       FluxEnsemble │               Latent │          0.0 │
│      mmd │   model │        overlapping │ LogisticRegression │              Generic │          0.0 │
│      mmd │   model │        overlapping │ LogisticRegression │ Generic_conservative │        0.066 │
│      mmd │   model │        overlapping │ LogisticRegression │             ClapROAR │        0.178 │
│      mmd │   model │        overlapping │ LogisticRegression │        Gravitational │        0.156 │
│      mmd │   model │        overlapping │ LogisticRegression │               Latent │        0.006 │
│      mmd │   model │        overlapping │          FluxModel │              Generic │        0.004 │
│      mmd │   model │        overlapping │          FluxModel │ Generic_conservative │         0.13 │
│      mmd │   model │        overlapping │          FluxModel │             ClapROAR │        0.486 │
│      mmd │   model │        overlapping │          FluxModel │        Gravitational │        0.564 │
│      mmd │   model │        overlapping │          FluxModel │               Latent │        0.046 │
│      mmd │   model │        overlapping │       FluxEnsemble │              Generic │          0.0 │
│      mmd │   model │        overlapping │       FluxEnsemble │ Generic_conservative │        0.122 │
│      mmd │   model │        overlapping │       FluxEnsemble │             ClapROAR │         0.46 │
│      mmd │   model │        overlapping │       FluxEnsemble │        Gravitational │        0.514 │
│      mmd │   model │        overlapping │       FluxEnsemble │               Latent │        0.048 │
│      mmd │   model │ linearly_separable │ LogisticRegression │              Generic │          0.0 │
│      mmd │   model │ linearly_separable │ LogisticRegression │ Generic_conservative │        0.376 │
│      mmd │   model │ linearly_separable │ LogisticRegression │             ClapROAR │        0.916 │
│      mmd │   model │ linearly_separable │ LogisticRegression │        Gravitational │        0.904 │
│      mmd │   model │ linearly_separable │ LogisticRegression │               Latent │        0.428 │
│      mmd │   model │ linearly_separable │          FluxModel │              Generic │        0.916 │
│      mmd │   model │ linearly_separable │          FluxModel │ Generic_conservative │        0.968 │
│      mmd │   model │ linearly_separable │          FluxModel │             ClapROAR │        0.958 │
│      mmd │   model │ linearly_separable │          FluxModel │        Gravitational │        0.982 │
│      mmd │   model │ linearly_separable │          FluxModel │               Latent │         0.83 │
│      mmd │   model │ linearly_separable │       FluxEnsemble │              Generic │        0.922 │
│      mmd │   model │ linearly_separable │       FluxEnsemble │ Generic_conservative │        0.968 │
│      mmd │   model │ linearly_separable │       FluxEnsemble │             ClapROAR │        0.962 │
│      mmd │   model │ linearly_separable │       FluxEnsemble │        Gravitational │        0.976 │
│      mmd │   model │ linearly_separable │       FluxEnsemble │               Latent │        0.862 │
│      mmd │   model │            circles │ LogisticRegression │              Generic │          0.8 │
│      mmd │   model │            circles │ LogisticRegression │ Generic_conservative │        0.996 │
│      mmd │   model │            circles │ LogisticRegression │             ClapROAR │        0.996 │
│      mmd │   model │            circles │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │   model │            circles │ LogisticRegression │               Latent │          0.0 │
│      mmd │   model │            circles │          FluxModel │              Generic │          1.0 │
│      mmd │   model │            circles │          FluxModel │ Generic_conservative │          1.0 │
│      mmd │   model │            circles │          FluxModel │             ClapROAR │        0.998 │
│      mmd │   model │            circles │          FluxModel │        Gravitational │        0.986 │
│      mmd │   model │            circles │          FluxModel │               Latent │        0.998 │
│      mmd │   model │            circles │       FluxEnsemble │              Generic │        0.998 │
│      mmd │   model │            circles │       FluxEnsemble │ Generic_conservative │        0.998 │
│      mmd │   model │            circles │       FluxEnsemble │             ClapROAR │        0.998 │
│      mmd │   model │            circles │       FluxEnsemble │        Gravitational │        0.978 │
│      mmd │   model │            circles │       FluxEnsemble │               Latent │          1.0 │
│      mmd │   model │              moons │ LogisticRegression │              Generic │          0.0 │
│      mmd │   model │              moons │ LogisticRegression │ Generic_conservative │        0.086 │
│      mmd │   model │              moons │ LogisticRegression │             ClapROAR │        0.462 │
│      mmd │   model │              moons │ LogisticRegression │        Gravitational │        0.446 │
│      mmd │   model │              moons │ LogisticRegression │               Latent │        0.008 │
│      mmd │   model │              moons │          FluxModel │              Generic │        0.812 │
│      mmd │   model │              moons │          FluxModel │ Generic_conservative │         0.87 │
│      mmd │   model │              moons │          FluxModel │             ClapROAR │        0.956 │
│      mmd │   model │              moons │          FluxModel │        Gravitational │        0.984 │
│      mmd │   model │              moons │          FluxModel │               Latent │         0.94 │
│      mmd │   model │              moons │       FluxEnsemble │              Generic │        0.822 │
│      mmd │   model │              moons │       FluxEnsemble │ Generic_conservative │        0.818 │
│      mmd │   model │              moons │       FluxEnsemble │             ClapROAR │        0.966 │
│      mmd │   model │              moons │       FluxEnsemble │        Gravitational │       0.9775 │
│      mmd │   model │              moons │       FluxEnsemble │               Latent │        0.922 │
│ mmd_grid │   model │        overlapping │ LogisticRegression │              Generic │          0.0 │
│ mmd_grid │   model │        overlapping │ LogisticRegression │ Generic_conservative │        0.104 │
│ mmd_grid │   model │        overlapping │ LogisticRegression │             ClapROAR │        0.256 │
│ mmd_grid │   model │        overlapping │ LogisticRegression │        Gravitational │         0.27 │
│ mmd_grid │   model │        overlapping │ LogisticRegression │               Latent │         0.01 │
│ mmd_grid │   model │        overlapping │          FluxModel │              Generic │          0.0 │
│ mmd_grid │   model │        overlapping │          FluxModel │ Generic_conservative │        0.116 │
│ mmd_grid │   model │        overlapping │          FluxModel │             ClapROAR │        0.298 │
│ mmd_grid │   model │        overlapping │          FluxModel │        Gravitational │        0.344 │
│ mmd_grid │   model │        overlapping │          FluxModel │               Latent │        0.204 │
│ mmd_grid │   model │        overlapping │       FluxEnsemble │              Generic │        0.002 │
│ mmd_grid │   model │        overlapping │       FluxEnsemble │ Generic_conservative │        0.154 │
│ mmd_grid │   model │        overlapping │       FluxEnsemble │             ClapROAR │       0.4075 │
│ mmd_grid │   model │        overlapping │       FluxEnsemble │        Gravitational │        0.356 │
│ mmd_grid │   model │        overlapping │       FluxEnsemble │               Latent │        0.324 │
│ mmd_grid │   model │ linearly_separable │ LogisticRegression │              Generic │          0.0 │
│ mmd_grid │   model │ linearly_separable │ LogisticRegression │ Generic_conservative │          0.0 │
│ mmd_grid │   model │ linearly_separable │ LogisticRegression │             ClapROAR │        0.006 │
│ mmd_grid │   model │ linearly_separable │ LogisticRegression │        Gravitational │        0.342 │
│ mmd_grid │   model │ linearly_separable │ LogisticRegression │               Latent │          0.0 │
│ mmd_grid │   model │ linearly_separable │          FluxModel │              Generic │          0.0 │
│ mmd_grid │   model │ linearly_separable │          FluxModel │ Generic_conservative │          0.0 │
│ mmd_grid │   model │ linearly_separable │          FluxModel │             ClapROAR │          0.0 │
│ mmd_grid │   model │ linearly_separable │          FluxModel │        Gravitational │        0.668 │
│ mmd_grid │   model │ linearly_separable │          FluxModel │               Latent │          0.0 │
│ mmd_grid │   model │ linearly_separable │       FluxEnsemble │              Generic │          0.0 │
│ mmd_grid │   model │ linearly_separable │       FluxEnsemble │ Generic_conservative │          0.0 │
│ mmd_grid │   model │ linearly_separable │       FluxEnsemble │             ClapROAR │          0.0 │
│ mmd_grid │   model │ linearly_separable │       FluxEnsemble │        Gravitational │        0.408 │
│ mmd_grid │   model │ linearly_separable │       FluxEnsemble │               Latent │          0.0 │
│ mmd_grid │   model │            circles │ LogisticRegression │              Generic │        0.814 │
│ mmd_grid │   model │            circles │ LogisticRegression │ Generic_conservative │        0.988 │
│ mmd_grid │   model │            circles │ LogisticRegression │             ClapROAR │        0.996 │
│ mmd_grid │   model │            circles │ LogisticRegression │        Gravitational │          0.0 │
│ mmd_grid │   model │            circles │ LogisticRegression │               Latent │          0.0 │
│ mmd_grid │   model │            circles │          FluxModel │              Generic │        0.814 │
│ mmd_grid │   model │            circles │          FluxModel │ Generic_conservative │        0.932 │
│ mmd_grid │   model │            circles │          FluxModel │             ClapROAR │         0.99 │
│ mmd_grid │   model │            circles │          FluxModel │        Gravitational │        0.406 │
│ mmd_grid │   model │            circles │          FluxModel │               Latent │         0.99 │
│ mmd_grid │   model │            circles │       FluxEnsemble │              Generic │        0.886 │
│ mmd_grid │   model │            circles │       FluxEnsemble │ Generic_conservative │         0.84 │
│ mmd_grid │   model │            circles │       FluxEnsemble │             ClapROAR │        0.984 │
│ mmd_grid │   model │            circles │       FluxEnsemble │        Gravitational │         0.55 │
│ mmd_grid │   model │            circles │       FluxEnsemble │               Latent │        0.996 │
│ mmd_grid │   model │              moons │ LogisticRegression │              Generic │          0.0 │
│ mmd_grid │   model │              moons │ LogisticRegression │ Generic_conservative │         0.09 │
│ mmd_grid │   model │              moons │ LogisticRegression │             ClapROAR │        0.458 │
│ mmd_grid │   model │              moons │ LogisticRegression │        Gravitational │        0.456 │
│ mmd_grid │   model │              moons │ LogisticRegression │               Latent │        0.008 │
│ mmd_grid │   model │              moons │          FluxModel │              Generic │        0.016 │
│ mmd_grid │   model │              moons │          FluxModel │ Generic_conservative │         0.03 │
│ mmd_grid │   model │              moons │          FluxModel │             ClapROAR │        0.004 │
│ mmd_grid │   model │              moons │          FluxModel │        Gravitational │        0.426 │
│ mmd_grid │   model │              moons │          FluxModel │               Latent │        0.114 │
│ mmd_grid │   model │              moons │       FluxEnsemble │              Generic │          0.0 │
│ mmd_grid │   model │              moons │       FluxEnsemble │ Generic_conservative │        0.006 │
│ mmd_grid │   model │              moons │       FluxEnsemble │             ClapROAR │          0.0 │
│ mmd_grid │   model │              moons │       FluxEnsemble │        Gravitational │          0.4 │
│ mmd_grid │   model │              moons │       FluxEnsemble │               Latent │        0.344 │
└──────────┴─────────┴────────────────────┴────────────────────┴──────────────────────┴──────────────┘

5.4 Chart in paper

Figure 5.3 shows the chart that went into the paper.

Images.load(joinpath(www_artifact_path,"paper_synthetic_results.png"))

Figure 5.3: Chart in paper

5.5 Latent Space Search

generators = Dict(
    :Latent=>GenericGenerator(opt = opt, decision_threshold=0.5),
    :Latent_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
    :Gravitational=>GravitationalGenerator(opt = opt),
    :ClapROAR=>ClapROARGenerator(opt = opt)
)

experiments = set_up_experiments(data_sets,models,generators)

n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
T = 100
using Serialization
results = run_experiments(
    experiments;
    save_path=output_path, evaluate_every=evaluate_every, n_rounds=n_rounds, n_folds=n_folds, T=T, 
    latent_space = true
)
Serialization.serialize(joinpath(output_path,"results_synthetic_latent.jls"),results)

using Serialization
results = Serialization.deserialize(joinpath(output_path,"results_synthetic_latent.jls"))

using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
    plt = plot(res)
    Images.save(joinpath(www_path, "line_chart_latent_$(data_name).png"), plt)
    line_charts[data_name] = plt
    plt = plot(res,maximum(res.output.n))
    Images.save(joinpath(www_path, "errorbar_chart_latent_$(data_name).png"), plt)
    errorbar_charts[data_name] = plt
end

5.6 Plots

5.6.1 Line Charts

Figure 5.4 shows the evolution of the evaluation metrics over the course of the experiment.

img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart") .&& contains.(readdir(www_artifact_path),"latent")]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
    display(load(img))
end

(a) Circles

(b) Linearly Separable

(c) Moons

(d) Overlapping

Figure 5.4: Line Charts

5.6.2 Error Bar Charts

Figure 5.5 shows the evaluation metrics at the end of the experiments.

img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart") .&& contains.(readdir(www_artifact_path),"latent")]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
    display(load(img))
end

(a) Circles

(b) Linearly Separable

(c) Moons

(d) Overlapping

Figure 5.5: Error Bar Charts

5.7 Bootstrap

n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_latent.csv"))

5.8 Chart in paper

Figure 5.6 shows the chart that went into the paper.

Images.load(joinpath(www_artifact_path,"paper_synthetic_latent_results.png"))

Figure 5.6: Chart in paper

5.9 Real World

models = [
    :LogisticRegression, 
    :FluxModel, 
    :FluxEnsemble,
]
opt = Flux.Descent(0.01) 
generators = Dict(
    :Generic=>GenericGenerator(opt = opt, decision_threshold=0.5),
    :Latent=>REVISEGenerator(opt = opt),
    :Generic_conservative=>GenericGenerator(opt = opt, decision_threshold=0.9),
    :Gravitational=>GravitationalGenerator(opt = opt),
    :ClapROAR=>ClapROARGenerator(opt = opt)
)

max_obs = 2500
data_path = data_dir("real_world")
data_sets = load_real_world(max_obs)
choices = [
    :cal_housing, 
    :credit_default, 
    :gmsc, 
]
data_sets = filter(p -> p[1] in choices, data_sets)

using CounterfactualExplanations.DataPreprocessing: unpack
bs = 500
function data_loader(data::CounterfactualData)
    X, y = unpack(data)
    data = Flux.DataLoader((X,y),batchsize=bs)
    return data
end
model_params = (batch_norm=false,n_hidden=64,n_layers=3,dropout=true,p_dropout=0.1)

experiments = set_up_experiments(
    data_sets,models,generators; 
    pre_train_models=100, model_params=model_params, 
    data_loader=data_loader
)

n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
n_samples = 10000
T = 100
generative_model_params = (epochs=250, latent_dim=8)
results = run_experiments(
    experiments;
    save_path=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T, n_samples=n_samples,
    generative_model_params=generative_model_params
)
Serialization.serialize(joinpath(output_path,"results_real_world.jls"),results)

using Serialization
results = Serialization.deserialize(joinpath(output_path,"results_real_world.jls"))

using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
    plt = plot(res)
    Images.save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
    line_charts[data_name] = plt
    plt = plot(res,maximum(res.output.n))
    Images.save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
    errorbar_charts[data_name] = plt
end

5.9.1 Line Charts

Figure 4.1 shows the evolution of the evaluation metrics over the course of the experiment.

choices = [
    :cal_housing, 
    :credit_default, 
    :gmsc, 
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"line_chart")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
    display(load(img))
end

(a) California Housing

(b) Credit Default

(c) GMSC

Figure 5.7: Line Charts

5.9.2 Error Bar Charts

Figure 4.2 shows the evaluation metrics at the end of the experiments.

choices = [
    :cal_housing, 
    :credit_default, 
    :gmsc, 
]
img_files = readdir(www_artifact_path)[contains.(readdir(www_artifact_path),"errorbar_chart")]
img_files = img_files[Bool.(reduce(+, map(choice -> contains.(img_files, string(choice)), choices)))]
img_files = joinpath.(www_artifact_path,img_files)
for img in img_files
    display(load(img))
end

(a) California Housing

(b) Credit Default

(c) GMSC

Figure 5.8: Error Bar Charts

5.9.3 Bootstrap

n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap_real_world.csv"))

┌──────────┬─────────┬────────────────┬────────────────────┬──────────────────────┬──────────────┐
│     name │   scope │           data │              model │            generator │ p_value_mean │
│ String31 │ String7 │       String15 │           String31 │             String31 │      Float64 │
├──────────┼─────────┼────────────────┼────────────────────┼──────────────────────┼──────────────┤
│      mmd │  domain │ credit_default │          FluxModel │ Generic_conservative │          1.0 │
│      mmd │  domain │ credit_default │          FluxModel │              Generic │          1.0 │
│      mmd │  domain │ credit_default │          FluxModel │             ClapROAR │          1.0 │
│      mmd │  domain │ credit_default │          FluxModel │        Gravitational │          0.0 │
│      mmd │  domain │ credit_default │          FluxModel │               Latent │          0.0 │
│      mmd │  domain │ credit_default │ LogisticRegression │ Generic_conservative │          1.0 │
│      mmd │  domain │ credit_default │ LogisticRegression │              Generic │          1.0 │
│      mmd │  domain │ credit_default │ LogisticRegression │             ClapROAR │          1.0 │
│      mmd │  domain │ credit_default │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │  domain │ credit_default │ LogisticRegression │               Latent │          0.8 │
│      mmd │  domain │ credit_default │       FluxEnsemble │ Generic_conservative │          1.0 │
│      mmd │  domain │ credit_default │       FluxEnsemble │              Generic │          1.0 │
│      mmd │  domain │ credit_default │       FluxEnsemble │             ClapROAR │          1.0 │
│      mmd │  domain │ credit_default │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │  domain │ credit_default │       FluxEnsemble │               Latent │          0.0 │
│      mmd │  domain │    cal_housing │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │  domain │    cal_housing │          FluxModel │              Generic │          0.0 │
│      mmd │  domain │    cal_housing │          FluxModel │             ClapROAR │          0.0 │
│      mmd │  domain │    cal_housing │          FluxModel │        Gravitational │          0.0 │
│      mmd │  domain │    cal_housing │          FluxModel │               Latent │          0.0 │
│      mmd │  domain │    cal_housing │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │  domain │    cal_housing │ LogisticRegression │              Generic │          0.0 │
│      mmd │  domain │    cal_housing │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │  domain │    cal_housing │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │  domain │    cal_housing │ LogisticRegression │               Latent │          0.0 │
│      mmd │  domain │    cal_housing │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │  domain │    cal_housing │       FluxEnsemble │              Generic │          0.0 │
│      mmd │  domain │    cal_housing │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │  domain │    cal_housing │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │  domain │    cal_housing │       FluxEnsemble │               Latent │          0.0 │
│      mmd │  domain │           gmsc │          FluxModel │ Generic_conservative │         0.93 │
│      mmd │  domain │           gmsc │          FluxModel │              Generic │        0.932 │
│      mmd │  domain │           gmsc │          FluxModel │             ClapROAR │        0.214 │
│      mmd │  domain │           gmsc │          FluxModel │        Gravitational │          0.0 │
│      mmd │  domain │           gmsc │          FluxModel │               Latent │          0.0 │
│      mmd │  domain │           gmsc │ LogisticRegression │ Generic_conservative │        0.004 │
│      mmd │  domain │           gmsc │ LogisticRegression │              Generic │        0.856 │
│      mmd │  domain │           gmsc │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │  domain │           gmsc │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │  domain │           gmsc │ LogisticRegression │               Latent │          0.0 │
│      mmd │  domain │           gmsc │       FluxEnsemble │ Generic_conservative │        0.758 │
│      mmd │  domain │           gmsc │       FluxEnsemble │              Generic │        0.938 │
│      mmd │  domain │           gmsc │       FluxEnsemble │             ClapROAR │         0.15 │
│      mmd │  domain │           gmsc │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │  domain │           gmsc │       FluxEnsemble │               Latent │          0.0 │
│      mmd │   model │ credit_default │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │   model │ credit_default │          FluxModel │              Generic │          0.0 │
│      mmd │   model │ credit_default │          FluxModel │             ClapROAR │          0.0 │
│      mmd │   model │ credit_default │          FluxModel │        Gravitational │          0.0 │
│      mmd │   model │ credit_default │          FluxModel │               Latent │          0.0 │
│      mmd │   model │ credit_default │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │   model │ credit_default │ LogisticRegression │              Generic │          0.0 │
│      mmd │   model │ credit_default │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │   model │ credit_default │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │   model │ credit_default │ LogisticRegression │               Latent │          0.0 │
│      mmd │   model │ credit_default │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │   model │ credit_default │       FluxEnsemble │              Generic │          0.0 │
│      mmd │   model │ credit_default │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │   model │ credit_default │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │   model │ credit_default │       FluxEnsemble │               Latent │          0.0 │
│      mmd │   model │    cal_housing │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │   model │    cal_housing │          FluxModel │              Generic │          0.0 │
│      mmd │   model │    cal_housing │          FluxModel │             ClapROAR │          0.0 │
│      mmd │   model │    cal_housing │          FluxModel │        Gravitational │          0.0 │
│      mmd │   model │    cal_housing │          FluxModel │               Latent │          0.0 │
│      mmd │   model │    cal_housing │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │   model │    cal_housing │ LogisticRegression │              Generic │          0.0 │
│      mmd │   model │    cal_housing │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │   model │    cal_housing │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │   model │    cal_housing │ LogisticRegression │               Latent │          0.0 │
│      mmd │   model │    cal_housing │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │   model │    cal_housing │       FluxEnsemble │              Generic │          0.0 │
│      mmd │   model │    cal_housing │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │   model │    cal_housing │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │   model │    cal_housing │       FluxEnsemble │               Latent │          0.0 │
│      mmd │   model │           gmsc │          FluxModel │ Generic_conservative │          0.0 │
│      mmd │   model │           gmsc │          FluxModel │              Generic │          0.0 │
│      mmd │   model │           gmsc │          FluxModel │             ClapROAR │          0.0 │
│      mmd │   model │           gmsc │          FluxModel │        Gravitational │          0.0 │
│      mmd │   model │           gmsc │          FluxModel │               Latent │          0.0 │
│      mmd │   model │           gmsc │ LogisticRegression │ Generic_conservative │          0.0 │
│      mmd │   model │           gmsc │ LogisticRegression │              Generic │          0.0 │
│      mmd │   model │           gmsc │ LogisticRegression │             ClapROAR │          0.0 │
│      mmd │   model │           gmsc │ LogisticRegression │        Gravitational │          0.0 │
│      mmd │   model │           gmsc │ LogisticRegression │               Latent │          0.0 │
│      mmd │   model │           gmsc │       FluxEnsemble │ Generic_conservative │          0.0 │
│      mmd │   model │           gmsc │       FluxEnsemble │              Generic │          0.0 │
│      mmd │   model │           gmsc │       FluxEnsemble │             ClapROAR │          0.0 │
│      mmd │   model │           gmsc │       FluxEnsemble │        Gravitational │          0.0 │
│      mmd │   model │           gmsc │       FluxEnsemble │               Latent │          0.0 │
│ mmd_grid │   model │ credit_default │          FluxModel │ Generic_conservative │          0.0 │
│ mmd_grid │   model │ credit_default │          FluxModel │              Generic │          0.0 │
│ mmd_grid │   model │ credit_default │          FluxModel │             ClapROAR │          0.0 │
│ mmd_grid │   model │ credit_default │          FluxModel │        Gravitational │          0.0 │
│ mmd_grid │   model │ credit_default │          FluxModel │               Latent │          0.0 │
│ mmd_grid │   model │ credit_default │ LogisticRegression │ Generic_conservative │          0.0 │
│ mmd_grid │   model │ credit_default │ LogisticRegression │              Generic │          0.0 │
│ mmd_grid │   model │ credit_default │ LogisticRegression │             ClapROAR │          0.0 │
│ mmd_grid │   model │ credit_default │ LogisticRegression │        Gravitational │          0.0 │
│ mmd_grid │   model │ credit_default │ LogisticRegression │               Latent │        0.078 │
│ mmd_grid │   model │ credit_default │       FluxEnsemble │ Generic_conservative │          0.0 │
│ mmd_grid │   model │ credit_default │       FluxEnsemble │              Generic │          0.0 │
│ mmd_grid │   model │ credit_default │       FluxEnsemble │             ClapROAR │          0.0 │
│ mmd_grid │   model │ credit_default │       FluxEnsemble │        Gravitational │          0.0 │
│ mmd_grid │   model │ credit_default │       FluxEnsemble │               Latent │          0.0 │
│ mmd_grid │   model │    cal_housing │          FluxModel │ Generic_conservative │          0.0 │
│ mmd_grid │   model │    cal_housing │          FluxModel │              Generic │          0.0 │
│ mmd_grid │   model │    cal_housing │          FluxModel │             ClapROAR │        0.012 │
│ mmd_grid │   model │    cal_housing │          FluxModel │        Gravitational │       0.0625 │
│ mmd_grid │   model │    cal_housing │          FluxModel │               Latent │          0.0 │
│ mmd_grid │   model │    cal_housing │ LogisticRegression │ Generic_conservative │          0.0 │
│ mmd_grid │   model │    cal_housing │ LogisticRegression │              Generic │          0.0 │
│ mmd_grid │   model │    cal_housing │ LogisticRegression │             ClapROAR │        0.004 │
│ mmd_grid │   model │    cal_housing │ LogisticRegression │        Gravitational │        0.014 │
│ mmd_grid │   model │    cal_housing │ LogisticRegression │               Latent │        0.002 │
│ mmd_grid │   model │    cal_housing │       FluxEnsemble │ Generic_conservative │        0.002 │
│ mmd_grid │   model │    cal_housing │       FluxEnsemble │              Generic │          0.0 │
│ mmd_grid │   model │    cal_housing │       FluxEnsemble │             ClapROAR │        0.044 │
│ mmd_grid │   model │    cal_housing │       FluxEnsemble │        Gravitational │          0.0 │
│ mmd_grid │   model │    cal_housing │       FluxEnsemble │               Latent │          0.0 │
│ mmd_grid │   model │           gmsc │          FluxModel │ Generic_conservative │          0.0 │
│ mmd_grid │   model │           gmsc │          FluxModel │              Generic │          0.0 │
│ mmd_grid │   model │           gmsc │          FluxModel │             ClapROAR │          0.0 │
│ mmd_grid │   model │           gmsc │          FluxModel │        Gravitational │          0.0 │
│ mmd_grid │   model │           gmsc │          FluxModel │               Latent │          0.0 │
│ mmd_grid │   model │           gmsc │ LogisticRegression │ Generic_conservative │          0.0 │
│ mmd_grid │   model │           gmsc │ LogisticRegression │              Generic │          0.0 │
│ mmd_grid │   model │           gmsc │ LogisticRegression │             ClapROAR │          0.0 │
│ mmd_grid │   model │           gmsc │ LogisticRegression │        Gravitational │          0.0 │
│ mmd_grid │   model │           gmsc │ LogisticRegression │               Latent │          0.0 │
│ mmd_grid │   model │           gmsc │       FluxEnsemble │ Generic_conservative │          0.0 │
│ mmd_grid │   model │           gmsc │       FluxEnsemble │              Generic │          0.0 │
│ mmd_grid │   model │           gmsc │       FluxEnsemble │             ClapROAR │          0.0 │
│ mmd_grid │   model │           gmsc │       FluxEnsemble │        Gravitational │          0.0 │
│ mmd_grid │   model │           gmsc │       FluxEnsemble │               Latent │          0.0 │
└──────────┴─────────┴────────────────┴────────────────────┴──────────────────────┴──────────────┘

5.9.4 Chart in paper

Figure 5.9 shows the chart that went into the paper.

Images.load(joinpath(www_artifact_path,"paper_real_world_results.png"))

Figure 5.9: Chart in paper