diff --git a/.github/workflows/actions.yaml b/.github/workflows/actions.yaml new file mode 100644 index 0000000..7f75a70 --- /dev/null +++ b/.github/workflows/actions.yaml @@ -0,0 +1,29 @@ +name: Build + +on: [push] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - name: Checkout the ${{ github.repository }} repository + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[test] + - name: Test with Pytest + run: | + pytest --cov=vecstack --cov-report=term-missing tests + - name: Coveralls + uses: coverallsapp/github-action@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0381104 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,74 @@ +# Changelog + +### v0.5.2 -- November 1, 2025 -- Maintenance release + +Enhance doc-strings + +### v0.5.1 -- September 28, 2025 -- Maintenance release + +Set minimum scikit-learn version to 1.6.0. +Function `check_estimator` requires usage of `validate_data` which was introduced in 1.6.0 + +### v0.5.0 -- September 8, 2025 -- Maintenance release + +* Python 3.9+ +* Testing: pytest and pytest-cov +* CI: GitHub Actions + +* Scikit-learn API: + * Fixed `_set_params` method which was not resetting individual estimators in the `estimators` collection + +* Functional API + * Fixed saving OOF arrays in file + +### v0.4.0 -- August 12, 2019 + +Since v0.4.0 vecstack provides official support for Python 3.5 and higher only, +but still there is unofficial support for Python 2.7 and Python 3.4. +Please see [details](https://github.com/vecxoz/vecstack/blob/master/PY2.md). + +Scikit-learn API: +* Fixed #31. `sklearn.externals.six` deprecation +* Fixed #29. Out-of-memory in `np.random.choice` for very large ranges + +Functional API: +* Feature #18. Added support for N-dimensional input. Useful for convolutional nets. +* Added aliases for `mode` parameter values which correspond to respective `variant` parameter values of `StackingTransformer`: + * 'oof_pred_bag' == 'A' + * 'oof_pred' == 'B' + +### v0.3.0 -- April 6, 2018 + +Introducing Scikit-learn API: `StackingTransformer` + +* Standard transformer class with `fit` and `transform` methods +* Compatible with `Pipeline` and `FeatureUnion` + +### v0.2.2 -- February 23, 2018 + +* Fixed #5. Wrong behavior during sparse matrix processing +* Improved input data validation +* Improved sparse matrix processing + +### v0.2.1 -- January 24, 2018 -- Maintenance release + +* Minor modifications + +### v0.2 -- January 23, 2018 + +New features: + +* Classification with probabilities +* Modes: compute only what you need (only OOF, only predictions, both, etc.) +* Save resulting arrays and log with model parameters + +### v0.1 -- November 22, 2016 -- Initial release + +Features: + +* Functional stacking API +* Regression +* Classification with class labels +* Ordinary and stratified k-fold split +* User-defined metric +* User-defined transformations for target and prediction diff --git a/LICENSE.txt b/LICENSE.txt index 3d51e67..13d488c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ MIT License Vecstack. Python package for stacking (machine learning technique) -Copyright (c) 2016 vecxoz +Copyright (c) 2016-2025 Igor Ivanov Email: vecxoz@gmail.com Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/PY2.md b/PY2.md new file mode 100644 index 0000000..da2c587 --- /dev/null +++ b/PY2.md @@ -0,0 +1,23 @@ +### Python 3.x + +Since v0.4.0 vecstack provides official support for Python 3.5 and higher only, +but still there is unofficial support for Python 2.7 and Python 3.4. See details below. + +The reason for these changes is global movement in Python 3.x direction. +Vecstack depends on scikit-learn which has already stopped support for Python < 3.5. +Scikit-learn v0.20.x is the last version supporting Python 2.7 and Python 3.4. +Vecstack follows this direction as well. +Please see [python3statement.org](https://python3statement.org/) for more details. + +### Unofficial support for Python 2.7 and Python 3.4 + +You can still install and run latest vecstack on Python 2.7 and Python 3.4. +NOTE. It will require legacy versions of the following packages: +* numpy<1.17 +* scipy<1.3 +* scikit-learn>=0.18,<0.21 + +There is a dedicated branch on GitHub called `py2` with appropriate requirements in `setup.py`. +Installation: + +`pip install https://github.com/vecxoz/vecstack/archive/py2.zip` diff --git a/README.md b/README.md index 9501715..fc6a2d7 100644 --- a/README.md +++ b/README.md @@ -1,239 +1,444 @@ +[![PyPI version](https://img.shields.io/pypi/v/vecstack.svg?colorB=4cc61e)](https://pypi.python.org/pypi/vecstack) +[![PyPI license](https://img.shields.io/pypi/l/vecstack.svg)](https://github.com/vecxoz/vecstack/blob/master/LICENSE.txt) +[![Build status](https://github.com/vecxoz/vecstack/actions/workflows/actions.yaml/badge.svg?branch=master)](https://github.com/vecxoz/vecstack/actions) +[![Coverage Status](https://coveralls.io/repos/github/vecxoz/vecstack/badge.svg?branch=master)](https://coveralls.io/github/vecxoz/vecstack?branch=master) +[![PyPI pyversions](https://img.shields.io/pypi/pyversions/vecstack.svg)](https://pypi.python.org/pypi/vecstack/) + # vecstack -Python package for stacking (machine learning technique) -* Easy to use. Perform stacking in a [single line](https://github.com/vecxoz/vecstack#brief-example) -* Use any sklearn-like models -* Perform classification and regression tasks -* Apply any user-defined transformations for target and prediction -* Python 2, Python 3 -* Win, Linux, Mac -* [MIT license](https://github.com/vecxoz/vecstack/blob/master/LICENSE.txt) -* Depends on **numpy**, **scipy**, **scikit-learn** - -Below you can find -* [Installation](https://github.com/vecxoz/vecstack#installation) guide -* [Brief example](https://github.com/vecxoz/vecstack#brief-example) -* Complete examples for [regression](https://github.com/vecxoz/vecstack#regression) and [classification](https://github.com/vecxoz/vecstack#classification) -* Explanation of [**stacking concept**](https://github.com/vecxoz/vecstack#stacking-concept) with pictures - -You can also look at detailed [parameter description](https://github.com/vecxoz/vecstack/blob/master/vecstack/core.py#L61) or just type ```>>>help(stacking)``` +Python package for stacking (stacked generalization) featuring lightweight ***functional API*** and fully compatible ***scikit-learn API*** +Convenient way to automate OOF computation, prediction and bagging using any number of models + +* [Functional API](https://github.com/vecxoz/vecstack#usage-functional-api): + * Minimalistic. Get your stacked features in a single line + * RAM-friendly. The lowest possible memory consumption + * Kaggle-ready. Stacked features and hyperparameters from each run can be [automatically saved](https://github.com/vecxoz/vecstack/blob/master/vecstack/core.py#L210) in files. No more mess at the end of the competition. [Log example](https://github.com/vecxoz/vecstack/blob/master/examples/03_log_example.txt) +* [Scikit-learn API](https://github.com/vecxoz/vecstack#usage-scikit-learn-api): + * Standardized. Fully scikit-learn compatible transformer class exposing `fit` and `transform` methods + * Pipeline-certified. Implement and deploy [multilevel stacking](https://github.com/vecxoz/vecstack/blob/master/examples/04_sklearn_api_regression_pipeline.ipynb) like it's no big deal using `sklearn.pipeline.Pipeline` + * And of course `FeatureUnion` is also invited to the party +* Overall specs: + * Use any sklearn-like estimators + * Perform [classification and regression](https://github.com/vecxoz/vecstack/blob/master/vecstack/coresk.py#L87) tasks + * Predict [class labels or probabilities](https://github.com/vecxoz/vecstack/blob/master/vecstack/coresk.py#L123) in classification task + * Apply any [user-defined metric](https://github.com/vecxoz/vecstack/blob/master/vecstack/coresk.py#L128) + * Apply any [user-defined transformations](https://github.com/vecxoz/vecstack/blob/master/vecstack/coresk.py#L91) for target and prediction + * Python 3.9+, [unofficial support for Python 2.7 and 3.4](https://github.com/vecxoz/vecstack/blob/master/PY2.md) + * Win, Linux, Mac + * [MIT license](https://github.com/vecxoz/vecstack/blob/master/LICENSE.txt) + * Depends on **numpy**, **scipy**, **scikit-learn>=1.6.0** + +# Get started +* [FAQ](https://github.com/vecxoz/vecstack#stacking-faq) +* [Installation guide](https://github.com/vecxoz/vecstack#installation) +* Usage: + * [Functional API](https://github.com/vecxoz/vecstack#usage-functional-api) + * [Scikit-learn API](https://github.com/vecxoz/vecstack#usage-scikit-learn-api) +* Tutorials: + * [Stacking concept + Pictures + Stacking implementation from scratch](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb) +* Examples (all examples are valid for both API with little [difference in parameters](https://github.com/vecxoz/vecstack#21-how-do-parameters-of-stacking-function-and-stackingtransformer-correspond)): + * Functional API: + * [Regression](https://github.com/vecxoz/vecstack/blob/master/examples/01_regression.ipynb) + * [Classification with class labels](https://github.com/vecxoz/vecstack/blob/master/examples/02_classification_with_class_labels.ipynb) + * [Classification with probabilities + Detailed workflow](https://github.com/vecxoz/vecstack/blob/master/examples/03_classification_with_proba_detailed_workflow.ipynb) + * Scikit-learn API: + * [Regression + Multilevel stacking using Pipeline](https://github.com/vecxoz/vecstack/blob/master/examples/04_sklearn_api_regression_pipeline.ipynb) +* Documentation: + * [Functional API](https://github.com/vecxoz/vecstack/blob/master/vecstack/core.py#L133) or type ```>>> help(stacking)``` + * [Scikit-learn API](https://github.com/vecxoz/vecstack/blob/master/vecstack/coresk.py#L68) or type ```>>> help(StackingTransformer)``` # Installation -**1-st way** -* From the command line run ```pip install vecstack``` - -**2-nd way** -* Download package file from [PyPI](https://pypi.python.org/pypi/vecstack) -* Unpack archive -* From the command line go to unpacked directory -* Run ```python setup.py install``` - -**3-rd way** -* Download package file from [PyPI](https://pypi.python.org/pypi/vecstack) -* From the command line go to the directory where you've downloaded package file -* Run ```python -m pip install vecstack-0.1.zip``` - -# Brief example +***Note:*** Python 3.9+ is officially supported and tested. If you’re still using Python 2.7 or 3.4 see [installation details here](https://github.com/vecxoz/vecstack/blob/master/PY2.md) + +* ***Classic 1st time installation (recommended):*** + * `pip install vecstack` +* Install for current user only (if you have some troubles with write permission): + * `pip install --user vecstack` +* If your PATH doesn't work: + * `/usr/bin/python -m pip install vecstack` + * `C:/Python3/python -m pip install vecstack` +* Upgrade vecstack and all dependencies: + * `pip install --upgrade vecstack` +* Upgrade vecstack WITHOUT upgrading dependencies: + * `pip install --upgrade --no-deps vecstack` +* Upgrade directly from GitHub WITHOUT upgrading dependencies: + * `pip install --upgrade --no-deps https://github.com/vecxoz/vecstack/archive/master.zip` +* Uninstall + * `pip uninstall vecstack` + +# Usage. Functional API ```python from vecstack import stacking # Get your data -# Initialize 1-st level models +# Initialize 1st level estimators +models = [LinearRegression(), + Ridge(random_state=0)] -# Get your stacking features in a single line -S_train, S_test = stacking(models, X_train, y_train, X_test, regression = True, verbose = 2) +# Get your stacked features in a single line +S_train, S_test = stacking(models, X_train, y_train, X_test, regression=True, verbose=2) -# Use 2-nd level model with stacking features +# Use 2nd level estimator with stacked features ``` -# Complete examples +# Usage. Scikit-learn API +```python +from vecstack import StackingTransformer -## Regression +# Get your data -```python -from sklearn.datasets import load_boston -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_absolute_error -from sklearn.ensemble import ExtraTreesRegressor -from sklearn.ensemble import RandomForestRegressor -from xgboost import XGBRegressor -from vecstack import stacking +# Initialize 1st level estimators +estimators = [('lr', LinearRegression()), + ('ridge', Ridge(random_state=0))] + +# Initialize StackingTransformer +stack = StackingTransformer(estimators, regression=True, verbose=2) + +# Fit +stack = stack.fit(X_train, y_train) + +# Get your stacked features +S_train = stack.transform(X_train) +S_test = stack.transform(X_test) -# Load demo data -boston = load_boston() -X, y = boston.data, boston.target - -# Make train/test split -# As usual in machine learning task we have X_train, y_train, and X_test -X_train, X_test, y_train, y_test = train_test_split(X, y, - test_size = 0.2, random_state = 0) - -# Caution! All models and parameter values are just -# demonstrational and shouldn't be considered as recommended. -# Initialize 1-st level models. -models = [ - ExtraTreesRegressor(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), - - RandomForestRegressor(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), - - XGBRegressor(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3)] - -# Compute stacking features -S_train, S_test = stacking(models, X_train, y_train, X_test, - regression = True, metric = mean_absolute_error, n_folds = 4, - shuffle = True, random_state = 0, verbose = 2) - -# Initialize 2-nd level model -model = XGBRegressor(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3) - -# Fit 2-nd level model -model = model.fit(S_train, y_train) - -# Predict -y_pred = model.predict(S_test) - -# Final prediction score -print('Final prediction score: [%.8f]' % mean_absolute_error(y_test, y_pred)) +# Use 2nd level estimator with stacked features ``` +# Stacking FAQ + +1. [How can I report an issue? How can I ask a question about stacking or vecstack package?](https://github.com/vecxoz/vecstack#1-how-can-i-report-an-issue-how-can-i-ask-a-question-about-stacking-or-vecstack-package) +2. [How can I say thanks?](https://github.com/vecxoz/vecstack#2-how-can-i-say-thanks) +3. [How to cite vecstack?](https://github.com/vecxoz/vecstack#3-how-to-cite-vecstack) +4. [What is stacking?](https://github.com/vecxoz/vecstack#4-what-is-stacking) +5. [What about stacking name?](https://github.com/vecxoz/vecstack#5-what-about-stacking-name) +6. [Do I need stacking at all?](https://github.com/vecxoz/vecstack#6-do-i-need-stacking-at-all) +7. [Can you explain stacking (stacked generalization) in 10 lines of code?](https://github.com/vecxoz/vecstack#7-can-you-explain-stacking-stacked-generalization-in-10-lines-of-code) +8. [Why do I need complicated inner procedure for stacking?](https://github.com/vecxoz/vecstack#8-why-do-i-need-complicated-inner-procedure-for-stacking) +9. [I want to implement stacking (stacked generalization) from scratch. Can you help me?](https://github.com/vecxoz/vecstack#9-i-want-to-implement-stacking-stacked-generalization-from-scratch-can-you-help-me) +10. [What is OOF?](https://github.com/vecxoz/vecstack#10-what-is-oof) +11. [What are *estimator*, *learner*, *model*?](https://github.com/vecxoz/vecstack#11-what-are-estimator-learner-model) +12. [What is *blending*? How is it related to stacking?](https://github.com/vecxoz/vecstack#12-what-is-blending-how-is-it-related-to-stacking) +13. [How to optimize weights for weighted average?](https://github.com/vecxoz/vecstack#13-how-to-optimize-weights-for-weighted-average) +14. [What is better: weighted average for current level or additional level?](https://github.com/vecxoz/vecstack#14-what-is-better-weighted-average-for-current-level-or-additional-level) +15. [What is *bagging*? How is it related to stacking?](https://github.com/vecxoz/vecstack#15-what-is-bagging-how-is-it-related-to-stacking) +16. [How many models should I use on a given stacking level?](https://github.com/vecxoz/vecstack#16-how-many-models-should-i-use-on-a-given-stacking-level) +17. [How many stacking levels should I use?](https://github.com/vecxoz/vecstack#17-how-many-stacking-levels-should-i-use) +18. [How do I choose models for stacking?](https://github.com/vecxoz/vecstack#18-how-do-i-choose-models-for-stacking) +19. [I am trying hard but still can't beat my best single model with stacking. What is wrong?](https://github.com/vecxoz/vecstack#19-i-am-trying-hard-but-still-cant-beat-my-best-single-model-with-stacking-what-is-wrong) +20. [What should I choose: functional API (`stacking` function) or Scikit-learn API (`StackingTransformer`)?](https://github.com/vecxoz/vecstack#20-what-should-i-choose-functional-api-stacking-function-or-scikit-learn-api-stackingtransformer) +21. [How do parameters of `stacking` function and `StackingTransformer` correspond?](https://github.com/vecxoz/vecstack#21-how-do-parameters-of-stacking-function-and-stackingtransformer-correspond) +22. [Why Scikit-learn API was implemented as transformer and not predictor?](https://github.com/vecxoz/vecstack#22-why-scikit-learn-api-was-implemented-as-transformer-and-not-predictor) +23. [How to estimate stacking training time and number of models which will be built?](https://github.com/vecxoz/vecstack#23-how-to-estimate-stacking-training-time-and-number-of-models-which-will-be-built) +24. [Which stacking variant should I use: 'A' ('oof_pred_bag') or 'B' ('oof_pred')?](https://github.com/vecxoz/vecstack#24-which-stacking-variant-should-i-use-a-oof_pred_bag-or-b-oof_pred) +25. [How to choose number of folds?](https://github.com/vecxoz/vecstack#25-how-to-choose-number-of-folds) +26. [When I transform train set I see 'Train set was detected'. What does it mean?](https://github.com/vecxoz/vecstack#26-when-i-transform-train-set-i-see-train-set-was-detected-what-does-it-mean) +27. [How is the very first stacking level called: L0 or L1? Where does counting start?](https://github.com/vecxoz/vecstack#27-how-is-the-very-first-stacking-level-called-l0-or-l1-where-does-counting-start) +28. [Can I use `(Randomized)GridSearchCV` to tune the whole stacking Pipeline?](https://github.com/vecxoz/vecstack#28-can-i-use-randomizedgridsearchcv-to-tune-the-whole-stacking-pipeline) +29. [How to define custom metric, especially AUC?](https://github.com/vecxoz/vecstack#29-how-to-define-custom-metric-especially-auc) +30. [Do folds (splits) have to be the same across estimators and stacking levels? How does `random_state` work?](https://github.com/vecxoz/vecstack#30-do-folds-splits-have-to-be-the-same-across-estimators-and-stacking-levels-how-does-random_state-work) +31. [How does `vecstack.StackingTransformer` differ from `sklearn.ensemble.StackingClassifier`?](https://github.com/vecxoz/vecstack#31-how-does-vecstackstackingtransformer-differ-from-sklearnensemblestackingclassifier) + +### 1. How can I report an issue? How can I ask a question about stacking or vecstack package? + +Just open an issue [here](https://github.com/vecxoz/vecstack/issues). +Ask me anything on the topic. +I'm a bit busy, so typically I answer on the next day. + +### 2. How can I say thanks? + +Just give me a star in the top right corner of the repository page. + +### 3. How to cite vecstack? + ``` -task: [regression] -metric: [mean_absolute_error] - -model 0: [ExtraTreesRegressor] - fold 0: [3.20733439] - fold 1: [2.87943130] - fold 2: [2.53026486] - fold 3: [2.83618694] - ---- - MEAN: [2.86330437] - -model 1: [RandomForestRegressor] - fold 0: [3.11110485] - fold 1: [2.78404210] - fold 2: [2.55707729] - fold 3: [2.32209992] - ---- - MEAN: [2.69358104] - -model 2: [XGBRegressor] - fold 0: [2.40318939] - fold 1: [2.37286982] - fold 2: [1.89121530] - fold 3: [1.95382831] - ---- - MEAN: [2.15527571] - -Final prediction score: [2.78409065] +@misc{vecstack2016, + author = {Igor Ivanov}, + title = {Vecstack}, + year = {2016}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/vecxoz/vecstack}}, +} ``` -## Classification +### 4. What is stacking? -```python -from sklearn.datasets import load_iris -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score -from sklearn.ensemble import ExtraTreesClassifier -from sklearn.ensemble import RandomForestClassifier -from xgboost import XGBClassifier -from vecstack import stacking +Stacking (stacked generalization) is a machine learning ensembling technique. +Main idea is to use predictions as features. +More specifically we predict train set (in CV-like fashion) and test set using some 1st level model(s), and then use these predictions as features for 2nd level model. You can find more details (concept, pictures, code) in [stacking tutorial](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb). +Also make sure to check out: +* [Ensemble Learning](https://en.wikipedia.org/wiki/Ensemble_learning) ([Stacking](https://en.wikipedia.org/wiki/Ensemble_learning#Stacking)) in Wikipedia +* Classical [Kaggle Ensembling Guide](https://mlwave.com/kaggle-ensembling-guide/) or try [another link](https://web.archive.org/web/20210727094233/https://mlwave.com/kaggle-ensembling-guide/) +* [Stacked Generalization](https://www.researchgate.net/publication/222467943_Stacked_Generalization) paper by David H. Wolpert + +### 5. What about stacking name? + +Often it is also called *stacked generalization*. The term is derived from the verb *to stack* (to put together, to put on top of each other). It implies that we put some models on top of other models, i.e. train some models on predictions of other models. From another point of view we can say that we stack predictions in order to use them as features. + +### 6. Do I need stacking at all? + +It depends on specific business case. The main thing to know about stacking is that it requires ***significant computing resources***. [No Free Lunch Theorem](https://en.wikipedia.org/wiki/There_ain%27t_no_such_thing_as_a_free_lunch) applies as always. Stacking can give you an improvement but for certain price (deployment, computation, maintenance). Only experiment for given business case will give you an answer: is it worth an effort and money. + +At current point large part of stacking users are participants of machine learning competitions. On Kaggle you can't go too far without ensembling. I can secretly tell you that at least top half of leaderboard in pretty much any competition uses ensembling (stacking) in some way. Stacking is less popular in production due to time and resource constraints, but I think it gains popularity. + +### 7. Can you explain stacking (stacked generalization) in 10 lines of code? + +[Of course](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb) + +### 8. Why do I need complicated inner procedure for stacking? + +I can just do the following. Why not? -# Load demo data -iris = load_iris() -X, y = iris.data, iris.target - -# Make train/test split -# As usual in machine learning task we have X_train, y_train, and X_test -X_train, X_test, y_train, y_test = train_test_split(X, y, - test_size = 0.2, random_state = 0) - -# Caution! All models and parameter values are just -# demonstrational and shouldn't be considered as recommended. -# Initialize 1-st level models. -models = [ - ExtraTreesClassifier(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), - - RandomForestClassifier(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), - - XGBClassifier(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3)] - -# Compute stacking features -S_train, S_test = stacking(models, X_train, y_train, X_test, - regression = False, metric = accuracy_score, n_folds = 4, - stratified = True, shuffle = True, random_state = 0, verbose = 2) - -# Initialize 2-nd level model -model = XGBClassifier(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3) - -# Fit 2-nd level model -model = model.fit(S_train, y_train) - -# Predict -y_pred = model.predict(S_test) - -# Final prediction score -print('Final prediction score: [%.8f]' % accuracy_score(y_test, y_pred)) +```python +model_L1 = XGBRegressor() +model_L1 = model_L1.fit(X_train, y_train) +S_train = model_L1.predict(X_train).reshape(-1, 1) # <- DOES NOT work due to overfitting. Must be CV +S_test = model_L1.predict(X_test).reshape(-1, 1) +model_L2 = LinearRegression() +model_L2 = model_L2.fit(S_train, y_train) +final_prediction = model_L2.predict(S_test) ``` +Code above will give meaningless result. If we fit on `X_train` we can’t just predict `X_train`, because our 1st level model has already seen `X_train`, and its prediction will be overfitted. To avoid overfitting we perform cross-validation procedure and in each fold we predict out-of-fold (OOF) part of `X_train`. You can find more details (concept, pictures, code) in [stacking tutorial](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb). + +### 9. I want to implement stacking (stacked generalization) from scratch. Can you help me? + +[Not a problem](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb) + +### 10. What is OOF? + +OOF is abbreviation for out-of-fold prediction. It's also known as *OOF features*, *stacked features*, *stacking features*, etc. Basically it means predictions for the part of train data that model haven't seen during training. + +### 11. What are *estimator*, *learner*, *model*? + +Basically it is the same thing meaning *machine learning algorithm*. Often these terms are used interchangeably. +Speaking about inner stacking mechanics, you should remember that when you have *single 1st level model* there will be at least `n_folds` separate models *trained in each CV fold* on different subsets of data. See [Q23](https://github.com/vecxoz/vecstack#23-how-to-estimate-stacking-training-time-and-number-of-models-which-will-be-built) for more details. + +### 12. What is *blending*? How is it related to stacking? + +Basically it is the same thing. Both approaches use predictions as features. +Often these terms are used interchangeably. +The difference is how we generate features (predictions) for the next level: +* *stacking*: perform cross-validation procedure and predict each part of train set (OOF) +* *blending*: predict fixed holdout set + +*vecstack* package supports only *stacking* i.e. cross-validation approach. For given `random_state` value (e.g. 42) folds (splits) will be the same across all estimators. See also [Q30](https://github.com/vecxoz/vecstack#30-do-folds-splits-have-to-be-the-same-across-estimators-and-stacking-levels-how-does-random_state-work). + +### 13. How to optimize weights for weighted average? + +You can use for example: + +* `scipy.optimize.minimize` +* `scipy.optimize.differential_evolution` + +### 14. What is better: weighted average for current level or additional level? + +By default you can start from weighted average. It is easier to apply and more chances that it will give good result. Then you can try additional level which potentially can outperform weighted average (but not always and not in an easy way). Experiment is your friend. + +### 15. What is *bagging*? How is it related to stacking? + +[Bagging](https://en.wikipedia.org/wiki/Bootstrap_aggregating) or Bootstrap aggregating works as follows: generate subsets of training set, train models on these subsets and then find average of predictions. +Also term *bagging* is often used to describe following approaches: +* train several different models on the same data and average predictions +* train same model with different random seeds on the same data and average predictions + +So if we run stacking and just average predictions - it is *bagging*. + +### 16. How many models should I use on a given stacking level? + +***Note 1:*** The best architecture can be found only by experiment. +***Note 2:*** Always remember that higher number of levels or models does NOT guarantee better result. The key to success in stacking (and ensembling in general) is diversity - low correlation between models. + +It depends on many factors like type of problem, type of data, quality of models, correlation of models, expected result, etc. +Some example configurations are listed below. +* Reasonable starting point: + * `L1: 2-10 models -> L2: weighted (rank) average or single model` +* Then try to add more 1st level models and additional level: + * `L1: 10-50 models -> L2: 2-10 models -> L3: weighted (rank) average` +* If you're crunching numbers at Kaggle and decided to go wild: + * `L1: 100-inf models -> L2: 10-50 models -> L3: 2-10 models -> L4: weighted (rank) average` + +You can also find some winning stacking architectures on [Kaggle blog](http://blog.kaggle.com/), e.g.: [1st place in Homesite Quote Conversion](http://blog.kaggle.com/2016/04/08/homesite-quote-conversion-winners-write-up-1st-place-kazanova-faron-clobber/). + +### 17. How many stacking levels should I use? + +***Note 1:*** The best architecture can be found only by experiment. +***Note 2:*** Always remember that higher number of levels or models does NOT guarantee better result. The key to success in stacking (and ensembling in general) is diversity - low correlation between models. + +For some example configurations see [Q16](https://github.com/vecxoz/vecstack#16-how-many-models-should-i-use-on-a-given-stacking-level). + +### 18. How do I choose models for stacking? + +Based on experiments and correlation (e.g. Pearson). Less correlated models give better result. It means that we should never judge our models by accuracy only. We should also consider correlation (how given model is different from others). Sometimes inaccurate but very different model can add substantial value to resulting ensemble. + +### 19. I am trying hard but still can't beat my best single model with stacking. What is wrong? + +Nothing is wrong. Stacking is advanced complicated technique. It's hard to make it work. ***Solution:*** make sure to try weighted (rank) average first instead of additional level with some advanced models. Average is much easier to apply and in most cases it will surely outperform your best model. If still no luck - then probably your models are highly correlated. + +### 20. What should I choose: functional API (`stacking` function) or Scikit-learn API (`StackingTransformer`)? + +Quick guide: + +* By default start from `StackingTransformer` with familiar scikit-learn interface and logic +* If you need low RAM consumption try `stacking` function but remember that it does not store models and does not have scikit-learn capabilities + +Stacking API comparison: + +| **Property** | **stacking function** | **StackingTransformer** | +|----------------|:---------------------:|:-----------------------:| +| Execution time | Same | Same | +| RAM | Consumes the ***smallest possible amount of RAM***. Does not store models. At any point in time only one model is alive. Logic: train model -> predict -> delete -> etc. When execution ends all RAM is released.| Consumes ***much more RAM***. It stores all models built in each fold. This price is paid for standard scikit-learn capabilities like `Pipeline` and `FeatureUnion`. | +| Access to models after training | No | Yes | +| Compatibility with `Pipeline` and `FeatureUnion` | No | Yes | +| Estimator implementation restrictions | Must have only `fit` and `predict` (`predict_proba`) methods | Must be fully scikit-learn compatible | +| `NaN` and `inf` in input data | Allowed | Not allowed | +| Can automatically save OOF and log in files | Yes | No | +| Input dimensionality (`X_train`, `X_test`) | Arbitrary | 2-D | + +### 21. How do parameters of `stacking` function and `StackingTransformer` correspond? + +| **stacking function** | **StackingTransformer** | +|---------------------------------------|-----------------------------------| +| `models=[Ridge()]` | `estimators=[('ridge', Ridge())]` | +| `mode='oof_pred_bag'` (alias `'A'`) | `variant='A'` | +| `mode='oof_pred'` (alias `'B'`) | `variant='B'` | + +### 22. Why Scikit-learn API was implemented as transformer and not predictor? + +* By nature stacking procedure is predictor, but ***by application*** it is definitely transformer. +* Transformer architecture was chosen because first of all user needs direct access to OOF. I.e. the ability to compute correlations, weighted average, etc. +* If you need predictor based on `StackingTransformer` you can easily create it via `Pipeline` by adding on the top of `StackingTransformer` some regressor or classifier. +* Transformer makes it easy to create any number of stacking levels. Using Pipeline we can easily create multilevel stacking by just adding several `StackingTransformer`'s on top of each other and then some final regressor or classifier. + +### 23. How to estimate stacking training time and number of models which will be built? + +***Note:*** Stacking usually takes long time. It's expected (inevitable) behavior. + +We can compute total number of models which will be built during stacking procedure using following formulas: +* Variant A: `n_models_total = n_estimators * n_folds` +* Variant B: `n_models_total = n_estimators * n_folds + n_estimators` + +Let's look at example. Say we define our stacking procedure as follows: +```python +estimators_L1 = [('lr', LinearRegression()), + ('ridge', Ridge())] +stack = StackingTransformer(estimators_L1, n_folds=4) ``` -task: [classification] -metric: [accuracy_score] - -model 0: [ExtraTreesClassifier] - fold 0: [0.93548387] - fold 1: [0.96666667] - fold 2: [1.00000000] - fold 3: [0.89655172] - ---- - MEAN: [0.95000000] - -model 1: [RandomForestClassifier] - fold 0: [0.87096774] - fold 1: [0.96666667] - fold 2: [1.00000000] - fold 3: [0.93103448] - ---- - MEAN: [0.94166667] - -model 2: [XGBClassifier] - fold 0: [0.83870968] - fold 1: [0.93333333] - fold 2: [1.00000000] - fold 3: [0.93103448] - ---- - MEAN: [0.92500000] - -Final prediction score: [0.96666667] +So we have two 1st level estimators and 4 folds. It means stacking procedure will build the following number of models: +* Variant A: 8 models total. Each model is trained on 3/4 of `X_train`. +* Variant B: 10 models total. 8 models are trained on 3/4 of `X_train` and 2 models on full `X_train`. + +Compute time: +* If estimators have relatively *similar training time*, we can roughly compute total training time as: `time_total = n_models_total * time_of_one_model` +* If estimators have *different training time*, we should compute number of models and time for each estimator separately (set `n_estimators=1` in formulas above) and then sum up times. + +### 24. Which stacking variant should I use: 'A' ('oof_pred_bag') or 'B' ('oof_pred')? + +You can find out only by experiment. Default choice is variant ***A***, because it takes ***less time*** and there should be no significant difference in result. But of course you may also try variant B. For more details see [stacking tutorial](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb). + +### 25. How to choose number of folds? + +***Note:*** Remember that higher number of folds substantially increase training time (and RAM consumption for StackingTransformer). See [Q23](https://github.com/vecxoz/vecstack#23-how-to-estimate-stacking-training-time-and-number-of-models-which-will-be-built). + +* Standard approach: 4 or 5 folds. +* If data is big: 3 folds. +* If data is small: you can try more folds like 10 or so. + +### 26. When I transform train set I see 'Train set was detected'. What does it mean? + +***Note 1:*** It is ***NOT allowed to change train set*** between calls to `fit` and `transform` methods. Due to stacking nature transformation is different for train set and any other set. If train set is changed after training, stacking procedure will not be able to correctly identify it and transformation will be wrong. + +***Note 2:*** To be correctly detected train set does not necessarily have to be identical (exactly the same). It must have the same shape and all values must be *close* (`np.isclose` is used for checking). So if you somehow regenerate your train set you should not worry about numerical precision. + +If you transform `X_train` and see 'Train set was detected' everything is OK. If you transform `X_train` but you don't see this message then something went wrong. Probably your train set was changed (it is not allowed). In this case you have to retrain `StackingTransformer`. For more details see [stacking tutorial](https://github.com/vecxoz/vecstack/blob/master/examples/00_stacking_concept_pictures_code.ipynb) or [Q8](https://github.com/vecxoz/vecstack#8-why-do-i-need-complicated-inner-procedure-for-stacking). + +### 27. How is the very first stacking level called: L0 or L1? Where does counting start? + +***Common convention:*** The very first bunch of models which are trained on initial raw data are called ***L1***. On top of L1 we have so called *stacker level* or *meta level* or L2 i.e. models which are trained on predictions of L1 models. Count continues in the same fashion up to arbitrary number of levels. + +I use this convention in my code and docs. But of course your Kaggle teammates may use some other naming approach, so you should clarify this for your specific case. + +### 28. Can I use `(Randomized)GridSearchCV` to tune the whole stacking Pipeline? + +Yes, technically you can, but it ***is not recommended*** because this approach will lead to redundant computations. General practical advice is to ***tune each estimator separately*** and then use tuned estimators on the 1st level of stacking. Higher level estimators should be tuned in the same fashion using OOF from previous level. For manual tuning you can use `stacking` function or `StackingTransformer` with a single 1st level estimator. + +### 29. How to define custom metric, especially AUC? + +```python +from sklearn.metrics import roc_auc_score +from sklearn.preprocessing import OneHotEncoder + +def auc(y_true, y_pred): + """ROC AUC metric for both binary and multiclass classification. + + Parameters + ---------- + y_true : 1d numpy array + True class labels + y_pred : 2d numpy array + Predicted probabilities for each class + """ + ohe = OneHotEncoder(sparse=False) + y_true = ohe.fit_transform(y_true.reshape(-1, 1)) + auc_score = roc_auc_score(y_true, y_pred) + return auc_score ``` +### 30. Do folds (splits) have to be the same across estimators and stacking levels? How does `random_state` work? + +To ensure better result, folds (splits) have to be the same across all estimators and all stacking levels. It means that `random_state` has to be the same in every call to `stacking` function or `StackingTransformer`. This is default behavior of `stacking` function and `StackingTransformer` (by default `random_state=0`). If you want to try different folds (splits) try to set different `random_state` values. + +### 31. How does `vecstack.StackingTransformer` differ from `sklearn.ensemble.StackingClassifier`? + +It significantly differs. Please see a [detailed explanation](https://github.com/vecxoz/vecstack/issues/37). + + # Stacking concept -1. We want to predict train and test sets with some 1-st level model(s), and then use this predictions as features for 2-nd level model. -2. Any model can be used as 1-st level model or 2-nd level model. -3. To avoid overfitting (for train set) we use cross-validation technique and in each fold we predict out-of-fold part of train set. +1. We want to predict train set and test set with some 1st level model(s), and then use these predictions as features for 2nd level model(s). +2. Any model can be used as 1st level model or 2nd level model. +3. To avoid overfitting (for train set) we use cross-validation technique and in each fold we predict out-of-fold (OOF) part of train set. 4. The common practice is to use from 3 to 10 folds. -5. In each fold we predict full test set, so after completion of all folds we need to find mean (mode) of all test set predictions made in each fold. -6. As an example we look at stacking implemented with single 1-st level model and 3-fold cross-validation. -7. Tree pictures below describe three folds of cross-validation. After completion of all three folds we get single train feature and single test feature to use with 2-nd level model. -8. We can repeat this cycle using other 1-st level models to get more features for 2-nd level model. -9. At the bottom you can see [GIF animation](https://github.com/vecxoz/vecstack/blob/master/README.md#animation). +5. Predict test set: + * **Variant A:** In each fold we predict test set, so after completion of all folds we need to find mean (mode) of all temporary test set predictions made in each fold. + * **Variant B:** We do not predict test set during cross-validation cycle. After completion of all folds we perform additional step: fit model on full train set and predict test set once. This approach takes more time because we need to perform one additional fitting. +6. As an example we look at stacking implemented with single 1st level model and 3-fold cross-validation. +7. Pictures: + * **Variant A:** Three pictures describe three folds of cross-validation. After completion of all three folds we get single train feature and single test feature to use with 2nd level model. + * **Variant B:** First three pictures describe three folds of cross-validation (like in Variant A) to get single train feature and fourth picture describes additional step to get single test feature. +8. We can repeat this cycle using other 1st level models to get more features for 2nd level model. +9. You can also look at animation of [Variant A](https://github.com/vecxoz/vecstack#variant-a-animation) and [Variant B](https://github.com/vecxoz/vecstack#variant-b-animation). + + +# Variant A +![Fold 1 of 3](https://github.com/vecxoz/vecstack/raw/master/pic/dia1.png "Fold 1 of 3") *** -![stack1](https://github.com/vecxoz/vecstack/raw/master/pic/dia1.png "Fold 1 of 3") +![Fold 2 of 3](https://github.com/vecxoz/vecstack/raw/master/pic/dia2.png "Fold 2 of 3") +*** +![Fold 3 of 3](https://github.com/vecxoz/vecstack/raw/master/pic/dia3.png "Fold 3 of 3") + +# Variant A. Animation + +![Variant A. Animation](https://github.com/vecxoz/vecstack/raw/master/pic/animation1.gif "Variant A. Animation") + +# Variant B + +![Step 1 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia4.png "Step 1 of 4") *** -![stack2](https://github.com/vecxoz/vecstack/raw/master/pic/dia2.png "Fold 2 of 3") +![Step 2 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia5.png "Step 2 of 4") *** -![stack3](https://github.com/vecxoz/vecstack/raw/master/pic/dia3.png "Fold 3 of 3") +![Step 3 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia6.png "Step 3 of 4") *** +![Step 4 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia7.png "Step 4 of 4") + +# Variant B. Animation + +![Variant B. Animation](https://github.com/vecxoz/vecstack/raw/master/pic/animation2.gif "Variant B. Animation") + + +# References -# Animation -![animation](https://github.com/vecxoz/vecstack/raw/master/pic/dia.gif "Animation") +* [Ensemble Learning](https://en.wikipedia.org/wiki/Ensemble_learning) ([Stacking](https://en.wikipedia.org/wiki/Ensemble_learning#Stacking)) in Wikipedia +* Classical [Kaggle Ensembling Guide](https://mlwave.com/kaggle-ensembling-guide/) or try [another link](https://web.archive.org/web/20210727094233/https://mlwave.com/kaggle-ensembling-guide/) +* [Stacked Generalization](https://www.researchgate.net/publication/222467943_Stacked_Generalization) paper by David H. Wolpert diff --git a/examples/00_stacking_concept_pictures_code.ipynb b/examples/00_stacking_concept_pictures_code.ipynb new file mode 100644 index 0000000..61fb330 --- /dev/null +++ b/examples/00_stacking_concept_pictures_code.ipynb @@ -0,0 +1,836 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Stacking concept + Pictures + Code\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Stacking concept\n", + "***\n", + "\n", + "1. We want to predict train set and test set with some 1st level model(s), and then use these predictions as features for 2nd level model(s). \n", + "2. Any model can be used as 1st level model or 2nd level model.\n", + "3. To avoid overfitting (for train set) we use cross-validation technique and in each fold we predict out-of-fold (OOF) part of train set.\n", + "4. The common practice is to use from 3 to 10 folds.\n", + "5. Predict test set:\n", + " * **Variant A:** In each fold we predict test set, so after completion of all folds we need to find mean (mode) of all temporary test set predictions made in each fold. \n", + " * **Variant B:** We do not predict test set during cross-validation cycle. After completion of all folds we perform additional step: fit model on full train set and predict test set once. This approach takes more time because we need to perform one additional fitting.\n", + "6. As an example we look at stacking implemented with single 1st level model and 3-fold cross-validation.\n", + "7. Pictures:\n", + " * **Variant A:** Three pictures describe three folds of cross-validation. After completion of all three folds we get single train feature and single test feature to use with 2nd level model. \n", + " * **Variant B:** First three pictures describe three folds of cross-validation (like in Variant A) to get single train feature and fourth picture describes additional step to get single test feature.\n", + "8. We can repeat this cycle using other 1st level models to get more features for 2nd level model.\n", + "9. You can also look at animation of Variant A and Variant B." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Variant A\n", + "***\n", + "![Fold 1 of 3](https://github.com/vecxoz/vecstack/raw/master/pic/dia1.png \"Fold 1 of 3\")\n", + "***\n", + "![Fold 2 of 3](https://github.com/vecxoz/vecstack/raw/master/pic/dia2.png \"Fold 2 of 3\")\n", + "***\n", + "![Fold 3 of 3](https://github.com/vecxoz/vecstack/raw/master/pic/dia3.png \"Fold 3 of 3\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Variant A. Animation\n", + "***\n", + "\n", + "![Variant A. Animation](https://github.com/vecxoz/vecstack/raw/master/pic/animation1.gif \"Variant A. Animation\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Variant B\n", + "***\n", + "![Step 1 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia4.png \"Step 1 of 4\")\n", + "***\n", + "![Step 2 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia5.png \"Step 2 of 4\")\n", + "***\n", + "![Step 3 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia6.png \"Step 3 of 4\")\n", + "***\n", + "![Step 4 of 4](https://github.com/vecxoz/vecstack/raw/master/pic/dia7.png \"Step 4 of 4\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Variant B. Animation\n", + "***\n", + "\n", + "![Variant B. Animation](https://github.com/vecxoz/vecstack/raw/master/pic/animation2.gif \"Variant B. Animation\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Code\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from sklearn.datasets import fetch_california_housing\n", + "from sklearn.model_selection import cross_val_predict\n", + "from sklearn.model_selection import cross_val_score\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.metrics import mean_absolute_error\n", + "from sklearn.metrics import make_scorer\n", + "from sklearn.linear_model import LinearRegression\n", + "from vecstack import stacking" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Prepare data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "X, y = fetch_california_housing(return_X_y=True)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Stacking. Variant A. Regression task" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## 1. Implementation A from scratch" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0: [0.53428460]\n", + "fold 1: [0.53291416]\n", + "fold 2: [0.52626198]\n", + "\n", + "MEAN: [0.53115358] + [0.00350384]\n", + "FULL: [0.53115358]\n" + ] + } + ], + "source": [ + "# 1st level model\n", + "model = LinearRegression()\n", + "# Number of folds\n", + "n_folds = 3\n", + "# Empty array to store out-of-fold predictions (single column)\n", + "S_train_A_scratch = np.zeros((X_train.shape[0], 1))\n", + "# Empty array to store temporary test set predictions made in each fold\n", + "S_test_temp = np.zeros((X_test.shape[0], n_folds))\n", + "# Empty list to store scores from each fold\n", + "scores = []\n", + "# Split initialization\n", + "kf = KFold(n_splits=n_folds, shuffle=False)\n", + "\n", + "# Loop across folds\n", + "for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)):\n", + " \n", + " # Split data and target\n", + " X_tr = X_train[tr_index]\n", + " y_tr = y_train[tr_index]\n", + " X_te = X_train[te_index]\n", + " y_te = y_train[te_index]\n", + " \n", + " # Fit\n", + " _ = model.fit(X_tr, y_tr)\n", + " \n", + " # Predict out-of-fold part of train set\n", + " S_train_A_scratch[te_index, :] = model.predict(X_te).reshape(-1, 1)\n", + " \n", + " # Predict test set\n", + " S_test_temp[:, fold_counter] = model.predict(X_test)\n", + " \n", + " # Print score of current fold\n", + " score = mean_absolute_error(y_te, S_train_A_scratch[te_index, :])\n", + " scores.append(score)\n", + " print('fold %d: [%.8f]' % (fold_counter, score))\n", + " \n", + "# Compute mean of temporary test set predictions to get final test set prediction\n", + "S_test_A_scratch = np.mean(S_test_temp, axis=1).reshape(-1, 1)\n", + "\n", + "# Mean OOF score + std\n", + "print('\\nMEAN: [%.8f] + [%.8f]' % (np.mean(scores), np.std(scores)))\n", + "\n", + "# Full OOF score\n", + "# !!! FULL score slightly differs from MEAN score because folds contain\n", + "# different number of examples (404 can't be divided by 3)\n", + "# If we set n_folds=4 scores will be identical for given metric\n", + "print('FULL: [%.8f]' % (mean_absolute_error(y_train, S_train_A_scratch)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## 2. Implementation A using standard Scikit-learn tools" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": true + } + }, + "source": [ + "There are no suitable standard Scikit-learn tools for Variant A. \n", + "Please see corresponding implementation for Variant B below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## 3. Implementation A using vecstack" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [regression]\n", + "metric: [mean_absolute_error]\n", + "mode: [oof_pred_bag]\n", + "n_models: [1]\n", + "\n", + "model 0: [LinearRegression]\n", + " fold 0: [0.53428460]\n", + " fold 1: [0.53291416]\n", + " fold 2: [0.52626198]\n", + " ----\n", + " MEAN: [0.53115358] + [0.00350384]\n", + " FULL: [0.53115358]\n", + "\n" + ] + } + ], + "source": [ + "models = [LinearRegression()]\n", + "S_train_A_vecstack, S_test_A_vecstack = stacking(models, \n", + " X_train, y_train, X_test, \n", + " regression=True, \n", + " n_folds=n_folds,\n", + " mode='oof_pred_bag', \n", + " random_state=0, \n", + " verbose=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare results" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1.69019642]\n", + " [2.88448817]\n", + " [2.20976598]\n", + " [1.38160949]\n", + " [2.23334858]]\n", + "\n", + "[[1.69019642]\n", + " [2.88448817]\n", + " [2.20976598]\n", + " [1.38160949]\n", + " [2.23334858]]\n" + ] + } + ], + "source": [ + "print('%s\\n\\n%s' % (S_train_A_scratch[:5], S_train_A_vecstack[:5]))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[2.2792878 ]\n", + " [2.78905713]\n", + " [1.9027686 ]\n", + " [1.02009234]\n", + " [2.95248053]]\n", + "\n", + "[[2.2792878 ]\n", + " [2.78905713]\n", + " [1.9027686 ]\n", + " [1.02009234]\n", + " [2.95248053]]\n" + ] + } + ], + "source": [ + "print('%s\\n\\n%s' % (S_test_A_scratch[:5], S_test_A_vecstack[:5]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## Results from two implementations above are identical" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train:\n", + "\n", + "True\n", + "\n", + "Test:\n", + "\n", + "True\n" + ] + } + ], + "source": [ + "print('Train:\\n')\n", + "print(all(S_train_A_scratch == S_train_A_vecstack))\n", + "\n", + "print('\\nTest:\\n')\n", + "print(all(S_test_A_scratch == S_test_A_vecstack))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "# Stacking. Variant B. Regression task" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## 1. Implementation B from scratch" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0: [0.53428460]\n", + "fold 1: [0.53291416]\n", + "fold 2: [0.52626198]\n", + "\n", + "MEAN: [0.53115358] + [0.00350384]\n", + "FULL: [0.53115358]\n" + ] + } + ], + "source": [ + "# 1st level model\n", + "model = LinearRegression()\n", + "# Number of folds\n", + "n_folds = 3\n", + "# Empty array to store out-of-fold predictions (single column)\n", + "S_train_B_scratch = np.zeros((X_train.shape[0], 1))\n", + "# Empty list to store scores from each fold\n", + "scores = []\n", + "# Split initialization\n", + "kf = KFold(n_splits=n_folds, shuffle=False)\n", + "\n", + "# Loop across folds\n", + "for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)):\n", + " \n", + " # Split data and target\n", + " X_tr = X_train[tr_index]\n", + " y_tr = y_train[tr_index]\n", + " X_te = X_train[te_index]\n", + " y_te = y_train[te_index]\n", + " \n", + " # Fit\n", + " _ = model.fit(X_tr, y_tr)\n", + " \n", + " # Predict out-of-fold part of train set\n", + " S_train_B_scratch[te_index, :] = model.predict(X_te).reshape(-1, 1)\n", + " \n", + " # Print score of current fold\n", + " score = mean_absolute_error(y_te, S_train_B_scratch[te_index, :])\n", + " scores.append(score)\n", + " print('fold %d: [%.8f]' % (fold_counter, score))\n", + " \n", + "# Fit on full train set and predict test set once\n", + "_ = model.fit(X_train, y_train)\n", + "S_test_B_scratch = model.predict(X_test).reshape(-1, 1)\n", + "\n", + "# Mean OOF score + std\n", + "print('\\nMEAN: [%.8f] + [%.8f]' % (np.mean(scores), np.std(scores)))\n", + "\n", + "# Full OOF score\n", + "# !!! FULL score slightly differs from MEAN score because folds contain\n", + "# different number of examples (404 can't be divided by 3)\n", + "# If we set n_folds=4 scores will be identical for given metric\n", + "print('FULL: [%.8f]' % (mean_absolute_error(y_train, S_train_B_scratch)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## 2. Implementation B using standard Scikit-learn tools" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "fold 0: [0.53428460]\n", + "fold 1: [0.53291416]\n", + "fold 2: [0.52626198]\n", + "\n", + "MEAN: [0.53115358] + [0.00350384]\n", + "FULL: [0.53115358]\n" + ] + } + ], + "source": [ + "# 1st level model\n", + "model = LinearRegression()\n", + "\n", + "# Scorer for cross_val_score\n", + "scorer = make_scorer(mean_absolute_error)\n", + "\n", + "# Fit and predict out-of-fold parts of train set\n", + "S_train_B_sklearn = cross_val_predict(model, \n", + " X_train, y=y_train, \n", + " cv=n_folds, n_jobs=1, \n", + " verbose=0).reshape(-1, 1)\n", + "\n", + "# Fit on full train set and predict test set once\n", + "_ = model.fit(X_train, y_train)\n", + "S_test_B_sklearn = model.predict(X_test).reshape(-1, 1)\n", + "\n", + "# Compute scores\n", + "# !!! We need additional run of cross_val_score, because at current point \n", + "# cross_val_predict hasn't ability to show scores\n", + "scores = cross_val_score(model, \n", + " X_train, y=y_train, \n", + " cv=n_folds, n_jobs=1, \n", + " scoring=scorer, verbose=0)\n", + "\n", + "# Print score of each fold\n", + "for fold_counter, score in enumerate(scores):\n", + " print('fold %d: [%.8f]' % (fold_counter, score))\n", + " \n", + "# Mean OOF score + std\n", + "print('\\nMEAN: [%.8f] + [%.8f]' % (np.mean(scores), np.std(scores)))\n", + "\n", + "# Full OOF score\n", + "# !!! FULL score slightly differs from MEAN score because folds contain\n", + "# different number of examples (404 can't be divided by 3)\n", + "# If we set n_folds=4 scores will be identical for given metric\n", + "print('FULL: [%.8f]' % (mean_absolute_error(y_train, S_train_B_sklearn)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## 3. Implementation B using vecstack" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [regression]\n", + "metric: [mean_absolute_error]\n", + "mode: [oof_pred]\n", + "n_models: [1]\n", + "\n", + "model 0: [LinearRegression]\n", + " fold 0: [0.53428460]\n", + " fold 1: [0.53291416]\n", + " fold 2: [0.52626198]\n", + " ----\n", + " MEAN: [0.53115358] + [0.00350384]\n", + " FULL: [0.53115358]\n", + "\n", + " Fitting on full train set...\n", + "\n" + ] + } + ], + "source": [ + "models = [LinearRegression()]\n", + "S_train_B_vecstack, S_test_B_vecstack = stacking(models, \n", + " X_train, y_train, X_test, \n", + " regression=True, \n", + " n_folds=n_folds,\n", + " mode='oof_pred', \n", + " random_state=0, \n", + " verbose=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare results" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1.69019642]\n", + " [2.88448817]\n", + " [2.20976598]\n", + " [1.38160949]\n", + " [2.23334858]]\n", + "\n", + "[[1.69019642]\n", + " [2.88448817]\n", + " [2.20976598]\n", + " [1.38160949]\n", + " [2.23334858]]\n", + "\n", + "[[1.69019642]\n", + " [2.88448817]\n", + " [2.20976598]\n", + " [1.38160949]\n", + " [2.23334858]]\n" + ] + } + ], + "source": [ + "print('%s\\n\\n%s\\n\\n%s' % (S_train_B_scratch[:5], S_train_B_sklearn[:5], S_train_B_vecstack[:5]))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[2.28110738]\n", + " [2.79009128]\n", + " [1.90332794]\n", + " [1.01760331]\n", + " [2.94852425]]\n", + "\n", + "[[2.28110738]\n", + " [2.79009128]\n", + " [1.90332794]\n", + " [1.01760331]\n", + " [2.94852425]]\n", + "\n", + "[[2.28110738]\n", + " [2.79009128]\n", + " [1.90332794]\n", + " [1.01760331]\n", + " [2.94852425]]\n" + ] + } + ], + "source": [ + "print('%s\\n\\n%s\\n\\n%s' % (S_test_B_scratch[:5], S_test_B_sklearn[:5], S_test_B_vecstack[:5]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "deletable": true, + "editable": true + }, + "source": [ + "## Results from three implementations above are identical" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train:\n", + "\n", + "True\n", + "True\n", + "True\n", + "\n", + "Test:\n", + "\n", + "True\n", + "True\n", + "True\n" + ] + } + ], + "source": [ + "print('Train:\\n')\n", + "print(all(S_train_B_scratch == S_train_B_sklearn))\n", + "print(all(S_train_B_scratch == S_train_B_vecstack))\n", + "print(all(S_train_B_sklearn == S_train_B_vecstack))\n", + "\n", + "print('\\nTest:\\n')\n", + "print(all(S_test_B_scratch == S_test_B_sklearn))\n", + "print(all(S_test_B_scratch == S_test_B_vecstack))\n", + "print(all(S_test_B_sklearn == S_test_B_vecstack))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/01_regression.ipynb b/examples/01_regression.ipynb new file mode 100644 index 0000000..c6fc652 --- /dev/null +++ b/examples/01_regression.ipynb @@ -0,0 +1,270 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Regression\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import fetch_california_housing\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import mean_absolute_error\n", + "from sklearn.ensemble import ExtraTreesRegressor\n", + "from sklearn.ensemble import RandomForestRegressor\n", + "from xgboost import XGBRegressor\n", + "from vecstack import stacking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prepare data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "X, y = fetch_california_housing(return_X_y=True)\n", + "\n", + "# Make train/test split\n", + "# As usual in machine learning task we have X_train, y_train, and X_test\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialize 1st level models" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Caution! All models and parameter values are just \n", + "# demonstrational and shouldn't be considered as recommended.\n", + "models = [\n", + " ExtraTreesRegressor(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " RandomForestRegressor(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3)\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Perform stacking" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [regression]\n", + "metric: [mean_absolute_error]\n", + "mode: [oof_pred_bag]\n", + "n_models: [3]\n", + "\n", + "model 0: [ExtraTreesRegressor]\n", + " fold 0: [0.65383143]\n", + " fold 1: [0.65059961]\n", + " fold 2: [0.66233582]\n", + " fold 3: [0.64449777]\n", + " ----\n", + " MEAN: [0.65281616] + [0.00643746]\n", + " FULL: [0.65281616]\n", + "\n", + "model 1: [RandomForestRegressor]\n", + " fold 0: [0.58416160]\n", + " fold 1: [0.56449564]\n", + " fold 2: [0.57730149]\n", + " fold 3: [0.55014073]\n", + " ----\n", + " MEAN: [0.56902487] + [0.01298795]\n", + " FULL: [0.56902487]\n", + "\n", + "model 2: [XGBRegressor]\n", + " fold 0: [0.37287275]\n", + " fold 1: [0.36827074]\n", + " fold 2: [0.37315715]\n", + " fold 3: [0.36447933]\n", + " ----\n", + " MEAN: [0.36969499] + [0.00358177]\n", + " FULL: [0.36969499]\n", + "\n" + ] + } + ], + "source": [ + "S_train, S_test = stacking(models, # list of models\n", + " X_train, y_train, X_test, # data\n", + " regression=True, # regression task (if you need \n", + " # classification - set to False)\n", + " mode='oof_pred_bag', # mode: oof for train set, predict test \n", + " # set in each fold and find mean\n", + " save_dir=None, # do not save result and log (to save \n", + " # in current dir - set to '.')\n", + " metric=mean_absolute_error, # metric: callable\n", + " n_folds=4, # number of folds\n", + " shuffle=True, # shuffle the data\n", + " random_state=0, # ensure reproducibility\n", + " verbose=2) # print all info" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Look at the result\n", + "\n", + "So now we have OOF from 1st level models and we can build 2nd level model. \n", + "But first let's look at the result. \n", + "We have three 1st level models, so we expect to get three columns in `S_train` and `S_test`. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2.1381431 , 1.89449961, 1.85192811],\n", + " [2.29310757, 1.89309918, 2.92809105],\n", + " [2.07256939, 1.89449961, 2.10903692],\n", + " [1.51938275, 1.53835871, 1.37909698],\n", + " [1.93450337, 2.737813 , 3.23252964]])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_train[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2.12570438, 1.88503507, 1.57685581],\n", + " [2.57631542, 2.67168873, 2.70525175],\n", + " [2.06940157, 1.88669837, 1.69479051],\n", + " [1.64434775, 1.20196782, 0.96695787],\n", + " [2.33799194, 2.98206787, 3.8881467 ]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_test[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Apply 2nd level model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final prediction score: [0.35320658]\n" + ] + } + ], + "source": [ + "# Initialize 2nd level model\n", + "model = XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3)\n", + " \n", + "# Fit 2nd level model\n", + "model = model.fit(S_train, y_train)\n", + "\n", + "# Predict\n", + "y_pred = model.predict(S_test)\n", + "\n", + "# Final prediction score\n", + "print('Final prediction score: [%.8f]' % mean_absolute_error(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/02_classification_with_class_labels.ipynb b/examples/02_classification_with_class_labels.ipynb new file mode 100644 index 0000000..6b6d700 --- /dev/null +++ b/examples/02_classification_with_class_labels.ipynb @@ -0,0 +1,268 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Classification (with class labels)\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import accuracy_score\n", + "from sklearn.ensemble import ExtraTreesClassifier\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from xgboost import XGBClassifier\n", + "from vecstack import stacking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prepare data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "iris = load_iris()\n", + "X, y = iris.data, iris.target\n", + "\n", + "# Make train/test split\n", + "# As usual in machine learning task we have X_train, y_train, and X_test\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialize 1st level models" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Caution! All models and parameter values are just \n", + "# demonstrational and shouldn't be considered as recommended.\n", + "models = [\n", + " ExtraTreesClassifier(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " RandomForestClassifier(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " XGBClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3)\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Perform stacking" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [classification]\n", + "n_classes: [3]\n", + "metric: [accuracy_score]\n", + "mode: [oof_pred_bag]\n", + "n_models: [3]\n", + "\n", + "model 0: [ExtraTreesClassifier]\n", + " fold 0: [1.00000000]\n", + " fold 1: [0.90000000]\n", + " fold 2: [1.00000000]\n", + " fold 3: [0.90000000]\n", + " ----\n", + " MEAN: [0.95000000] + [0.05000000]\n", + " FULL: [0.95000000]\n", + "\n", + "model 1: [RandomForestClassifier]\n", + " fold 0: [0.96666667]\n", + " fold 1: [0.90000000]\n", + " fold 2: [1.00000000]\n", + " fold 3: [0.90000000]\n", + " ----\n", + " MEAN: [0.94166667] + [0.04330127]\n", + " FULL: [0.94166667]\n", + "\n", + "model 2: [XGBClassifier]\n", + " fold 0: [0.93333333]\n", + " fold 1: [0.90000000]\n", + " fold 2: [0.93333333]\n", + " fold 3: [0.90000000]\n", + " ----\n", + " MEAN: [0.91666667] + [0.01666667]\n", + " FULL: [0.91666667]\n", + "\n" + ] + } + ], + "source": [ + "S_train, S_test = stacking(models, # list of models\n", + " X_train, y_train, X_test, # data\n", + " regression=False, # classification task (if you need \n", + " # regression - set to True)\n", + " mode='oof_pred_bag', # mode: oof for train set, predict test \n", + " # set in each fold and vote\n", + " needs_proba=False, # predict class labels (if you need \n", + " # probabilities - set to True) \n", + " save_dir=None, # do not save result and log (to save \n", + " # in current dir - set to '.')\n", + " metric=accuracy_score, # metric: callable\n", + " n_folds=4, # number of folds\n", + " stratified=True, # stratified split for folds\n", + " shuffle=True, # shuffle the data\n", + " random_state=0, # ensure reproducibility\n", + " verbose=2) # print all info" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Look at the result\n", + "\n", + "So now we have OOF from 1st level models and we can build 2nd level model. \n", + "But first let's look at the result. \n", + "We have three 1st level models, so we expect to get three columns in `S_train` and `S_test`. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2, 2, 2],\n", + " [1, 1, 1],\n", + " [0, 0, 0],\n", + " [2, 2, 2],\n", + " [2, 2, 2]])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_train[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2, 2, 2],\n", + " [1, 1, 1],\n", + " [0, 0, 0],\n", + " [2, 2, 2],\n", + " [0, 0, 0]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_test[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Apply 2nd level model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final prediction score: [0.96666667]\n" + ] + } + ], + "source": [ + "# Initialize 2nd level model\n", + "model = XGBClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3)\n", + " \n", + "# Fit 2nd level model\n", + "model = model.fit(S_train, y_train)\n", + "\n", + "# Predict\n", + "y_pred = model.predict(S_test)\n", + "\n", + "# Final prediction score\n", + "print('Final prediction score: [%.8f]' % accuracy_score(y_test, y_pred))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/03_classification_with_proba_detailed_workflow.ipynb b/examples/03_classification_with_proba_detailed_workflow.ipynb new file mode 100644 index 0000000..a3b696a --- /dev/null +++ b/examples/03_classification_with_proba_detailed_workflow.ipynb @@ -0,0 +1,840 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Classification (with probabilities) + Detailed workflow\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-09-06 15:55:39.761844: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-09-06 15:55:39.792129: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2025-09-06 15:55:40.527567: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "W0000 00:00:1757163340.650380 23114 gpu_device.cc:2342] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n", + "Skipping registering GPU devices...\n" + ] + } + ], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore', message='The y_prob values do not sum to one')\n", + "warnings.filterwarnings('ignore', message='Skipping variable loading for optimizer')\n", + "warnings.filterwarnings('ignore', message='X does not have valid feature names')\n", + "import re\n", + "from glob import glob\n", + "import numpy as np\n", + "np.random.seed(0) # ensure reproducibility\n", + "np.set_printoptions(suppress=True)\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.datasets import make_classification\n", + "from sklearn.metrics import log_loss\n", + "# Models\n", + "from sklearn.naive_bayes import GaussianNB\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.ensemble import ExtraTreesClassifier\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from xgboost import XGBClassifier\n", + "from lightgbm import LGBMClassifier\n", + "# NN\n", + "import tensorflow as tf\n", + "# Data is small so do NOT use GPU for simplicity\n", + "tf.config.set_visible_devices([], 'GPU')\n", + "# Suppress Python level warnings from tensorflow\n", + "tf.get_logger().setLevel('ERROR') \n", + "from keras.models import Sequential\n", + "from keras.layers import Dense\n", + "from keras.layers import Input\n", + "from scikeras.wrappers import KerasClassifier, KerasRegressor\n", + "# Stacking\n", + "from vecstack import stacking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prepare data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train shape: (400, 5)\n", + "Test shape: (100, 5)\n" + ] + } + ], + "source": [ + "n_classes = 3\n", + "\n", + "# Create data: 500 example, 5 feature, 3 classes\n", + "X, y = make_classification(n_samples=500, n_features=5, \n", + " n_informative=3, n_redundant=1, \n", + " n_classes=n_classes, flip_y=0, \n", + " random_state=0)\n", + "\n", + "# Make train/test split\n", + "# As usual in machine learning task we have X_train, y_train, and X_test\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n", + "\n", + "print('Train shape:', X_train.shape)\n", + "print('Test shape: ', X_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialize 1st level models" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def build_keras_model_1():\n", + " model = Sequential()\n", + " model.add(Input(shape=(X_train.shape[1],)))\n", + " model.add(Dense(64,\n", + " kernel_initializer='normal', \n", + " activation='relu'))\n", + " model.add(Dense(n_classes, \n", + " kernel_initializer='normal', \n", + " activation='softmax'))\n", + " model.compile(optimizer='rmsprop', \n", + " loss='sparse_categorical_crossentropy', \n", + " metrics=['categorical_accuracy'])\n", + " return model\n", + "\n", + "# Caution! All models and parameter values are just \n", + "# demonstrational and shouldn't be considered as recommended.\n", + "models_1 = [ \n", + " GaussianNB(),\n", + " \n", + " LogisticRegression(random_state=0),\n", + " \n", + " ExtraTreesClassifier(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " RandomForestClassifier(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " XGBClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3),\n", + " \n", + " LGBMClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3, verbose=-1),\n", + "\n", + " KerasClassifier(model=build_keras_model_1(), epochs=2, \n", + " batch_size=32, verbose=0)\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Perform stacking" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [classification]\n", + "n_classes: [3]\n", + "metric: [log_loss]\n", + "mode: [oof_pred]\n", + "n_models: [7]\n", + "\n", + "model 0: [GaussianNB]\n", + " fold 0: [0.57030626]\n", + " fold 1: [0.28256165]\n", + " fold 2: [0.35609357]\n", + " fold 3: [0.57833219]\n", + " fold 4: [0.60933411]\n", + " ----\n", + " MEAN: [0.47932556] + [0.13332980]\n", + " FULL: [0.47932556]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "model 1: [LogisticRegression]\n", + " fold 0: [0.58074650]\n", + " fold 1: [0.27626266]\n", + " fold 2: [0.40797434]\n", + " fold 3: [0.48192230]\n", + " fold 4: [0.73155308]\n", + " ----\n", + " MEAN: [0.49569178] + [0.15420632]\n", + " FULL: [0.49569178]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "model 2: [ExtraTreesClassifier]\n", + " fold 0: [0.80523428]\n", + " fold 1: [0.72214703]\n", + " fold 2: [0.72851161]\n", + " fold 3: [0.78859646]\n", + " fold 4: [0.82512546]\n", + " ----\n", + " MEAN: [0.77392297] + [0.04137715]\n", + " FULL: [0.77392297]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "model 3: [RandomForestClassifier]\n", + " fold 0: [0.49744464]\n", + " fold 1: [0.30045852]\n", + " fold 2: [0.38866325]\n", + " fold 3: [0.49116102]\n", + " fold 4: [0.51274923]\n", + " ----\n", + " MEAN: [0.43809533] + [0.08160752]\n", + " FULL: [0.43809533]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "model 4: [XGBClassifier]\n", + " fold 0: [0.44344452]\n", + " fold 1: [0.19888857]\n", + " fold 2: [0.37032590]\n", + " fold 3: [0.50328894]\n", + " fold 4: [0.46400776]\n", + " ----\n", + " MEAN: [0.39599114] + [0.10760951]\n", + " FULL: [0.39599114]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "model 5: [LGBMClassifier]\n", + " fold 0: [0.46739007]\n", + " fold 1: [0.20192817]\n", + " fold 2: [0.34714699]\n", + " fold 3: [0.49378077]\n", + " fold 4: [0.50732813]\n", + " ----\n", + " MEAN: [0.40351483] + [0.11560251]\n", + " FULL: [0.40351483]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "model 6: [KerasClassifier]\n", + " fold 0: [0.99057170]\n", + " fold 1: [0.82460226]\n", + " fold 2: [0.70419501]\n", + " fold 3: [0.66333640]\n", + " fold 4: [0.66650450]\n", + " ----\n", + " MEAN: [0.76984197] + [0.12494150]\n", + " FULL: [0.76984197]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "Result was saved to [./[2025.09.06].[15.55.45].032643.7f0aac.npy]\n" + ] + } + ], + "source": [ + "S_train_1, S_test_1 = stacking(models_1, # list of models\n", + " X_train, y_train, X_test, # data\n", + " regression=False, # classification task (if you need \n", + " # regression - set to True)\n", + " mode='oof_pred', # mode: oof for train set, fit on full \n", + " # train and predict test set once\n", + " needs_proba=True, # predict probabilities (if you need \n", + " # class labels - set to False) \n", + " save_dir='.', # save result and log in current dir \n", + " # (to disable saving - set to None)\n", + " metric=log_loss, # metric: callable\n", + " n_folds=5, # number of folds\n", + " stratified=True, # stratified split for folds\n", + " shuffle=True, # shuffle the data\n", + " random_state=0, # ensure reproducibility\n", + " verbose=2) # print all info" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Look at the result" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "We have 3 classes and 7 models so in resulting arrays we expect to see 21 columns.\n", + "S_train_1 shape: (400, 21)\n", + "S_test_1 shape: (100, 21)\n" + ] + } + ], + "source": [ + "print('We have %d classes and %d models so in resulting arrays \\\n", + "we expect to see %d columns.' % (n_classes, len(models_1), n_classes * len(models_1)))\n", + "print('S_train_1 shape:', S_train_1.shape)\n", + "print('S_test_1 shape: ', S_test_1.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.00083161, 0.01828022, 0.98088816, 0.01605728, 0.02724415,\n", + " 0.95669857, 0.22361998, 0.20286901, 0.57351102, 0.0365039 ,\n", + " 0.10297998, 0.86051611, 0.00304293, 0.01768309, 0.97927397,\n", + " 0.00074538, 0.00395689, 0.99529773, 0.32723224, 0.28824005,\n", + " 0.38452768],\n", + " [0.95026182, 0.04973818, 0. , 0.89084281, 0.10909541,\n", + " 0.00006178, 0.55396899, 0.27694952, 0.16908149, 0.85772772,\n", + " 0.13275189, 0.00952038, 0.9821493 , 0.01676223, 0.00108847,\n", + " 0.99746299, 0.00249197, 0.00004503, 0.39017811, 0.32593027,\n", + " 0.28389156]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_train_1[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.38824186, 0.37434678, 0.23741136, 0.307547 , 0.52253329,\n", + " 0.16991971, 0.31662764, 0.29004533, 0.39332704, 0.27655712,\n", + " 0.55408115, 0.16936173, 0.58477622, 0.38952848, 0.02569526,\n", + " 0.72381327, 0.24995384, 0.02623288, 0.30865759, 0.34028247,\n", + " 0.35106 ],\n", + " [0.32313599, 0.67239959, 0.00446442, 0.28539557, 0.62320346,\n", + " 0.09140096, 0.31999925, 0.36345201, 0.31654874, 0.10054021,\n", + " 0.81354061, 0.08591918, 0.02924692, 0.95591789, 0.01483521,\n", + " 0.01189152, 0.94377175, 0.04433673, 0.27213532, 0.52776176,\n", + " 0.20010303]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_test_1[:2]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Our arrays and log were saved in current dir" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Arrays:\n", + "[2025.09.06].[15.55.45].032643.7f0aac.npy\n", + "\n", + "Logs:\n", + "[2025.09.06].[15.55.45].032643.7f0aac.log.txt\n" + ] + } + ], + "source": [ + "names = sorted(glob('*.npy'))\n", + "npy_1_name = names[0] # for later use\n", + "\n", + "print('Arrays:')\n", + "for name in names:\n", + " print(name)\n", + "\n", + "names = sorted(glob('*.log.txt'))\n", + "log_1_name = names[0] # for later use\n", + "\n", + "print('\\nLogs:')\n", + "for name in names:\n", + " print(name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialize some other 1st level model(s)\n", + "\n", + "As we continue to work on the problem we create many other models. \n", + "Let's say we want to try more powerful neural network." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def build_keras_model_2():\n", + " model = Sequential()\n", + " model.add(Input(shape=(X_train.shape[1],)))\n", + " model.add(Dense(256,\n", + " kernel_initializer='normal', \n", + " activation='relu'))\n", + " model.add(Dense(64, \n", + " kernel_initializer='normal', \n", + " activation='relu'))\n", + " model.add(Dense(n_classes, \n", + " kernel_initializer='normal', \n", + " activation='softmax'))\n", + " model.compile(optimizer='rmsprop', \n", + " loss='sparse_categorical_crossentropy', \n", + " metrics=['categorical_accuracy'])\n", + " return model\n", + "\n", + "# Caution! All models and parameter values are just \n", + "# demonstrational and shouldn't be considered as recommended.\n", + "models_2 = [ \n", + " KerasClassifier(model=build_keras_model_2(), epochs=5, \n", + " batch_size=32, verbose=0)\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Perform stacking again" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [classification]\n", + "n_classes: [3]\n", + "metric: [log_loss]\n", + "mode: [oof_pred]\n", + "n_models: [1]\n", + "\n", + "model 0: [KerasClassifier]\n", + " fold 0: [0.51636764]\n", + " fold 1: [0.22029691]\n", + " fold 2: [0.28535020]\n", + " fold 3: [0.39152662]\n", + " fold 4: [0.43074365]\n", + " ----\n", + " MEAN: [0.36885700] + [0.10502070]\n", + " FULL: [0.36885700]\n", + "\n", + " Fitting on full train set...\n", + "\n", + "Result was saved to [./[2025.09.06].[15.55.47].511377.c4aa2f.npy]\n" + ] + } + ], + "source": [ + "S_train_2, S_test_2 = stacking(models_2, # list of models\n", + " X_train, y_train, X_test, # data\n", + " regression=False, # classification task (if you need \n", + " # regression - set to True)\n", + " mode='oof_pred', # mode: oof for train set, fit on full \n", + " # train and predict test set once\n", + " needs_proba=True, # predict probabilities (if you need \n", + " # class labels - set to False) \n", + " save_dir='.', # save result and log in current dir \n", + " # (to disable saving - set to None)\n", + " metric=log_loss, # metric: callable\n", + " n_folds=5, # number of folds\n", + " stratified=True, # stratified split for folds\n", + " shuffle=True, # shuffle the data\n", + " random_state=0, # ensure reproducibility\n", + " verbose=2) # print all info" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# New arrays and log were saved too" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Arrays:\n", + "[2025.09.06].[15.55.45].032643.7f0aac.npy\n", + "[2025.09.06].[15.55.47].511377.c4aa2f.npy\n", + "\n", + "Logs:\n", + "[2025.09.06].[15.55.45].032643.7f0aac.log.txt\n", + "[2025.09.06].[15.55.47].511377.c4aa2f.log.txt\n" + ] + } + ], + "source": [ + "names = sorted(glob('*.npy'))\n", + "\n", + "print('Arrays:')\n", + "for name in names:\n", + " print(name)\n", + " \n", + "names = sorted(glob('*.log.txt'))\n", + "\n", + "print('\\nLogs:')\n", + "for name in names:\n", + " print(name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Time to collect results\n", + "\n", + "After several (many) days of building, optimizing, and testing models we have a lot of files with saved OOF. \n", + "At this point we can load and use OOF of specific model or all OOF we have." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Find specific model\n", + "\n", + "We can open logs and find the model of interest. \n", + "We can do it programmatically or just open logs in editor. \n", + "Name of the `.log.txt` file is the same as the name of corresponding `.npy` file (except extension). \n", + "To find columns containing OOF of specific model we use model index from log:\n", + "* if we predicted class labels - corresponding column index is just model index\n", + "* if we predicted probabilities - corresponding column index is model index multiplied by number of classes" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Let's open this log: [2025.09.06].[15.55.45].032643.7f0aac.log.txt\n", + "Let's look what models did we build in those session.\n", + "\n" + ] + } + ], + "source": [ + "print(\"Let's open this log: %s\" % log_1_name)\n", + "with open(log_1_name) as f:\n", + " lines = f.readlines()\n", + "\n", + "print(\"Let's look what models did we build in those session.\\n\")\n", + "for line in lines:\n", + " if re.search(r'^model [0-9]+', line):\n", + " print(line)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load specific model OOF\n", + "\n", + "Let's say we are interested in `LGBMClassifier`. \n", + "We found out that it has index 5. \n", + "Then we load target `.npy` file and because of probabilities we need 3 columns from 15 (5 \\* 3) to 18 (5 \\* 3 + 3)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Let's load this .npy file: [2025.09.06].[15.55.45].032643.7f0aac.npy\n" + ] + } + ], + "source": [ + "print(\"Let's load this .npy file: %s\" % npy_1_name)\n", + "S = np.load(npy_1_name, allow_pickle=True)\n", + "S_train_lgbm = S[0][:, 15:18]\n", + "S_test_lgbm = S[1][:, 15:18]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.00074538, 0.00395689, 0.99529773],\n", + " [0.99746299, 0.00249197, 0.00004503],\n", + " [0.99599212, 0.00369756, 0.00031032],\n", + " [0.00109104, 0.99659281, 0.00231615],\n", + " [0.98498265, 0.01489686, 0.00012049]])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_train_lgbm[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.72381327, 0.24995384, 0.02623288],\n", + " [0.01189152, 0.94377175, 0.04433673],\n", + " [0.0896902 , 0.90432675, 0.00598305],\n", + " [0.00034138, 0.99091816, 0.00874045],\n", + " [0.0001139 , 0.99955083, 0.00033527]])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_test_lgbm[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compute score of specific model" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LGBMCLassifier log loss: 0.40351483\n" + ] + } + ], + "source": [ + "print('LGBMCLassifier log loss: %.8f' % log_loss(y_train, S_train_lgbm))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load ALL OOF\n", + "\n", + "***Note:*** If you load OOF from scratch, don't forget to load `y_train` from initial dataset too." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "We have 3 classes and 8 models TOTAL so in resulting arrays we expect to see 24 columns.\n" + ] + } + ], + "source": [ + "print('We have %d classes and %d models TOTAL so in resulting arrays \\\n", + "we expect to see %d columns.' % (n_classes, len(models_1) + len(models_2), \n", + " n_classes * (len(models_1) + len(models_2))))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading: [2025.09.06].[15.55.45].032643.7f0aac.npy\n", + "Loading: [2025.09.06].[15.55.47].511377.c4aa2f.npy\n", + "\n", + "S_train_all shape: (400, 24)\n", + "S_test_all shape: (100, 24)\n" + ] + } + ], + "source": [ + "# Create empty arrays\n", + "S_train_all = np.zeros((X_train.shape[0], 0))\n", + "S_test_all = np.zeros((X_test.shape[0], 0))\n", + "\n", + "# Load results\n", + "for name in sorted(glob('*.npy')):\n", + " print('Loading: %s' % name)\n", + " S = np.load(name, allow_pickle=True)\n", + " S_train_all = np.c_[S_train_all, S[0]]\n", + " S_test_all = np.c_[S_test_all, S[1]]\n", + " \n", + "print('\\nS_train_all shape:', S_train_all.shape)\n", + "print('S_test_all shape: ', S_test_all.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Apply 2nd level model" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final prediction score: 0.37246788\n" + ] + } + ], + "source": [ + "# Initialize 2nd level model\n", + "model = XGBClassifier(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3)\n", + " \n", + "# Fit 2nd level model\n", + "model = model.fit(S_train_all, y_train)\n", + "\n", + "# Predict\n", + "y_pred = model.predict_proba(S_test_all)\n", + "\n", + "# Final prediction score\n", + "print('Final prediction score: %.8f' % log_loss(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/03_log_example.txt b/examples/03_log_example.txt new file mode 100644 index 0000000..cf7739a --- /dev/null +++ b/examples/03_log_example.txt @@ -0,0 +1,200 @@ +vecstack log [2018.02.01].[15.41.41].305268.0eadc0 + +task: [classification] +n_classes: [3] +metric: [log_loss] +mode: [oof_pred] +n_models: [7] + +---------------------------------------- +model 0: [GaussianNB] +---------------------------------------- + +priors None + + fold 0: [0.65551778] + fold 1: [0.42335961] + fold 2: [0.38132309] + fold 3: [0.57180128] + fold 4: [0.30426116] + ---- + MEAN: [0.46725258] + [0.12825825] + FULL: [0.46842847] + +---------------------------------------- +model 1: [LogisticRegression] +---------------------------------------- + +C 1.0 +class_weight None +dual False +fit_intercept True +intercept_scaling 1 +max_iter 100 +multi_class ovr +n_jobs 1 +penalty l2 +random_state 0 +solver liblinear +tol 0.0001 +verbose 0 +warm_start False + + fold 0: [0.59821304] + fold 1: [0.54202039] + fold 2: [0.55194968] + fold 3: [0.46887313] + fold 4: [0.44995007] + ---- + MEAN: [0.52220126] + [0.05499033] + FULL: [0.52280210] + +---------------------------------------- +model 2: [ExtraTreesClassifier] +---------------------------------------- + +bootstrap False +class_weight None +criterion gini +max_depth 3 +max_features auto +max_leaf_nodes None +min_impurity_split 1e-07 +min_samples_leaf 1 +min_samples_split 2 +min_weight_fraction_leaf 0.0 +n_estimators 100 +n_jobs -1 +oob_score False +random_state 0 +verbose 0 +warm_start False + + fold 0: [0.79961086] + fold 1: [0.75093790] + fold 2: [0.77930597] + fold 3: [0.76984042] + fold 4: [0.75288684] + ---- + MEAN: [0.77051640] + [0.01799067] + FULL: [0.77062834] + +---------------------------------------- +model 3: [RandomForestClassifier] +---------------------------------------- + +bootstrap True +class_weight None +criterion gini +max_depth 3 +max_features auto +max_leaf_nodes None +min_impurity_split 1e-07 +min_samples_leaf 1 +min_samples_split 2 +min_weight_fraction_leaf 0.0 +n_estimators 100 +n_jobs -1 +oob_score False +random_state 0 +verbose 0 +warm_start False + + fold 0: [0.61575788] + fold 1: [0.40598536] + fold 2: [0.37631065] + fold 3: [0.48927717] + fold 4: [0.35383760] + ---- + MEAN: [0.44823373] + [0.09551690] + FULL: [0.44901890] + +---------------------------------------- +model 4: [XGBClassifier] +---------------------------------------- + +base_score 0.5 +booster gbtree +colsample_bylevel 1 +colsample_bytree 1 +gamma 0 +learning_rate 0.1 +max_delta_step 0 +max_depth 3 +min_child_weight 1 +missing None +n_estimators 100 +n_jobs -1 +nthread None +objective binary:logistic +random_state 0 +reg_alpha 0 +reg_lambda 1 +scale_pos_weight 1 +seed None +silent True +subsample 1 + + fold 0: [0.79844773] + fold 1: [0.29219564] + fold 2: [0.32080725] + fold 3: [0.47567222] + fold 4: [0.28695253] + ---- + MEAN: [0.43481507] + [0.19447336] + FULL: [0.43610692] + +---------------------------------------- +model 5: [LGBMClassifier] +---------------------------------------- + +boosting_type gbdt +class_weight None +colsample_bytree 1.0 +learning_rate 0.1 +max_depth 3 +min_child_samples 20 +min_child_weight 0.001 +min_split_gain 0.0 +n_estimators 100 +n_jobs -1 +num_leaves 31 +objective None +random_state 0 +reg_alpha 0.0 +reg_lambda 0.0 +silent True +subsample 1.0 +subsample_for_bin 200000 +subsample_freq 1 + + fold 0: [0.72121715] + fold 1: [0.28661436] + fold 2: [0.32019533] + fold 3: [0.44805274] + fold 4: [0.29008710] + ---- + MEAN: [0.41323334] + [0.16487562] + FULL: [0.41430248] + +---------------------------------------- +model 6: [KerasClassifier] +---------------------------------------- + +batch_size 32 +build_fn +epochs 2 +verbose 0 + + fold 0: [0.95679030] + fold 1: [0.97577886] + fold 2: [0.99147471] + fold 3: [0.97480903] + fold 4: [0.96731000] + ---- + MEAN: [0.97323258] + [0.01137690] + FULL: [0.97322745] + +---------------------------------------- +END +---------------------------------------- diff --git a/examples/04_sklearn_api_regression_pipeline.ipynb b/examples/04_sklearn_api_regression_pipeline.ipynb new file mode 100644 index 0000000..502c1ee --- /dev/null +++ b/examples/04_sklearn_api_regression_pipeline.ipynb @@ -0,0 +1,693 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# StackingTransformer (scikit-learn API for stacking)\n", + "# Regression\n", + "# 2-level stacking: step-by-step and using Pipeline\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Import" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from sklearn.datasets import fetch_california_housing\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import mean_absolute_error\n", + "from sklearn.ensemble import ExtraTreesRegressor\n", + "from sklearn.ensemble import RandomForestRegressor\n", + "from sklearn.pipeline import Pipeline\n", + "import joblib\n", + "from xgboost import XGBRegressor\n", + "from vecstack import StackingTransformer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prepare data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "X, y = fetch_california_housing(return_X_y=True)\n", + "\n", + "# Make train/test split\n", + "# As usual in machine learning task we have X_train, y_train, and X_test\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialize 1st level estimators" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Caution! All estimators and parameter values are just \n", + "# demonstrational and shouldn't be considered as recommended.\n", + "\n", + "# This is list of tuples\n", + "# Each tuple contains arbitrary unique name and estimator object\n", + "estimators_L1 = [\n", + " ('et', ExtraTreesRegressor(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3)),\n", + " \n", + " ('rf', RandomForestRegressor(random_state=0, n_jobs=-1, \n", + " n_estimators=100, max_depth=3)),\n", + " \n", + " ('xgb', XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3))\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Initialize StackingTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "stack = StackingTransformer(estimators=estimators_L1, # base estimators\n", + " regression=True, # regression task (if you need \n", + " # classification - set to False)\n", + " variant='A', # oof for train set, predict test \n", + " # set in each fold and find mean\n", + " metric=mean_absolute_error, # metric: callable\n", + " n_folds=4, # number of folds\n", + " shuffle=True, # shuffle the data\n", + " random_state=0, # ensure reproducibility\n", + " verbose=2) # print all info" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Step-by-step approach" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fit StackingTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task: [regression]\n", + "metric: [mean_absolute_error]\n", + "variant: [A]\n", + "n_estimators: [3]\n", + "\n", + "estimator 0: [et: ExtraTreesRegressor]\n", + " fold 0: [0.65383143]\n", + " fold 1: [0.65059961]\n", + " fold 2: [0.66233582]\n", + " fold 3: [0.64449777]\n", + " ----\n", + " MEAN: [0.65281616] + [0.00643746]\n", + "\n", + "estimator 1: [rf: RandomForestRegressor]\n", + " fold 0: [0.58416160]\n", + " fold 1: [0.56449564]\n", + " fold 2: [0.57730149]\n", + " fold 3: [0.55014073]\n", + " ----\n", + " MEAN: [0.56902487] + [0.01298795]\n", + "\n", + "estimator 2: [xgb: XGBRegressor]\n", + " fold 0: [0.37287275]\n", + " fold 1: [0.36827074]\n", + " fold 2: [0.37315715]\n", + " fold 3: [0.36447933]\n", + " ----\n", + " MEAN: [0.36969499] + [0.00358177]\n", + "\n" + ] + } + ], + "source": [ + "stack = stack.fit(X_train, y_train)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get stacked features: transform (predict) train set and test set" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train set was detected.\n", + "Transforming...\n", + "\n", + "estimator 0: [et: ExtraTreesRegressor]\n", + " model from fold 0: done\n", + " model from fold 1: done\n", + " model from fold 2: done\n", + " model from fold 3: done\n", + " ----\n", + " DONE\n", + "\n", + "estimator 1: [rf: RandomForestRegressor]\n", + " model from fold 0: done\n", + " model from fold 1: done\n", + " model from fold 2: done\n", + " model from fold 3: done\n", + " ----\n", + " DONE\n", + "\n", + "estimator 2: [xgb: XGBRegressor]\n", + " model from fold 0: done\n", + " model from fold 1: done\n", + " model from fold 2: done\n", + " model from fold 3: done\n", + " ----\n", + " DONE\n", + "\n" + ] + } + ], + "source": [ + "S_train = stack.transform(X_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Transforming...\n", + "\n", + "estimator 0: [et: ExtraTreesRegressor]\n", + " model from fold 0: done\n", + " model from fold 1: done\n", + " model from fold 2: done\n", + " model from fold 3: done\n", + " ----\n", + " DONE\n", + "\n", + "estimator 1: [rf: RandomForestRegressor]\n", + " model from fold 0: done\n", + " model from fold 1: done\n", + " model from fold 2: done\n", + " model from fold 3: done\n", + " ----\n", + " DONE\n", + "\n", + "estimator 2: [xgb: XGBRegressor]\n", + " model from fold 0: done\n", + " model from fold 1: done\n", + " model from fold 2: done\n", + " model from fold 3: done\n", + " ----\n", + " DONE\n", + "\n" + ] + } + ], + "source": [ + "S_test = stack.transform(X_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Look at the result\n", + "\n", + "Let's look at our stacked features. \n", + "We have three 1st level estimators, so we expect to get three columns in `S_train` and `S_test`. " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2.1381431 , 1.89449961, 1.85192811],\n", + " [2.29310757, 1.89309918, 2.92809105],\n", + " [2.07256939, 1.89449961, 2.10903692],\n", + " [1.51938275, 1.53835871, 1.37909698],\n", + " [1.93450337, 2.737813 , 3.23252964]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_train[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[2.12570438, 1.88503507, 1.57685581],\n", + " [2.57631542, 2.67168873, 2.70525175],\n", + " [2.06940157, 1.88669837, 1.69479051],\n", + " [1.64434775, 1.20196782, 0.96695787],\n", + " [2.33799194, 2.98206787, 3.8881467 ]])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "S_test[:5]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Apply 2nd level estimator" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final prediction score: [0.35320658]\n" + ] + } + ], + "source": [ + "# Initialize 2nd level estimator\n", + "final_estimator = XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, \n", + " n_estimators=100, max_depth=3)\n", + "\n", + "# Fit\n", + "final_estimator = final_estimator.fit(S_train, y_train)\n", + "\n", + "# Predict\n", + "y_pred = final_estimator.predict(S_test)\n", + "\n", + "# Final prediction score\n", + "print('Final prediction score: [%.8f]' % mean_absolute_error(y_test, y_pred))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Some useful StackingTransformer attributes" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Number of base estimators\n", + "# Type: int\n", + "stack.n_estimators_" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[0.65383143, 0.65059961, 0.66233582, 0.64449777],\n", + " [0.5841616 , 0.56449564, 0.57730149, 0.55014073],\n", + " [0.37287275, 0.36827074, 0.37315715, 0.36447933]])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Scores for each estimator (rows) in each fold (columns)\n", + "# Type: 2d numpy array\n", + "stack.scores_" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('et', np.float64(0.6528161581671601), np.float64(0.006437456871932304)),\n", + " ('rf', np.float64(0.5690248659195717), np.float64(0.012987952596095562)),\n", + " ('xgb', np.float64(0.3696949910225933), np.float64(0.00358176792828805))]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Mean and std for each estimator\n", + "# Type: list of tuples\n", + "stack.mean_std_" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namemeanstd
2xgb0.3696950.003582
1rf0.5690250.012988
0et0.6528160.006437
\n", + "
" + ], + "text/plain": [ + " name mean std\n", + "2 xgb 0.369695 0.003582\n", + "1 rf 0.569025 0.012988\n", + "0 et 0.652816 0.006437" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Mean and std convenient representation using pandas.DataFrame\n", + "df = pd.DataFrame.from_records(stack.mean_std_, columns=['name', 'mean', 'std'])\n", + "# Sort by column 'mean' (best on the top)\n", + "df.sort_values('mean', ascending=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2. Pipeline\n", + "\n", + "StackingTransformer is fully scikit-learn compatible so we can easily implement **arbitrary number of stacking levels** using Pipeline\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Specify steps of Pipeline\n", + "steps = [('stack', stack),\n", + " ('final_estimator', final_estimator)]\n", + "\n", + "# Init Pipeline\n", + "pipe = Pipeline(steps)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# If we have several stacking levels our Pipeline steps would be:\n", + "# steps = [('stack_L1', stack_L1),\n", + "# ('stack_L2', stack_L2),\n", + "# ('stack_L99', stack_L99), # :-)\n", + "# ('final_estimator', final_estimator)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ability to set parameters of nested estimators and transformers\n", + "\n", + "Following scikit-learn naming convention we can access parameters of nested estimators and transformers. \n", + "Each nested level (should not be confused with stacking level) is separated by double underscore `__`. \n", + "For example using Pipeline we want StackingTransformer to be silent. \n", + "We can access StackingTransformer parameter `verbose` through `pipe` instance as `stack__verbose`. \n", + "We can also set any parameter of 1st level estimators inside StackingTransformer. \n", + "We can access XGBoost parameter `learning_rate` through `pipe` instance as `stack__xgb__learning_rate`." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "pipe = pipe.set_params(stack__verbose=0)\n", + "# pipe = pipe.set_params(stack__xgb__learning_rate=0.555)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fit and predict using Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final prediction score using Pipeline: [0.35320658]\n" + ] + } + ], + "source": [ + "# Fit\n", + "pipe = pipe.fit(X_train, y_train)\n", + "\n", + "# Predict\n", + "y_pred_pipe = pipe.predict(X_test)\n", + "\n", + "# Final prediction score\n", + "print('Final prediction score using Pipeline: [%.8f]' % mean_absolute_error(y_test, y_pred_pipe))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ability to save fitted Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "# Save Pipeline\n", + "_ = joblib.dump(pipe, 'pipe_with_stack.pkl')" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final prediction score using loaded Pipeline: [0.35320658]\n" + ] + } + ], + "source": [ + "# Load Pipeline\n", + "pipe_loaded = joblib.load('pipe_with_stack.pkl')\n", + "\n", + "# Predict using loaded Pipeline\n", + "y_pred_pipe_loaded = pipe_loaded.predict(X_test)\n", + "\n", + "# Final prediction score\n", + "print('Final prediction score using loaded Pipeline: [%.8f]' % mean_absolute_error(y_test, y_pred_pipe_loaded))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conclusion\n", + "\n", + "Step-by-step approach is useful when we need direct access to stacked features `S_train` and `S_test`. \n", + "Pipeline approach is useful when we need to represent many steps in a single object. \n", + "Both approaches give exactly the same score." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..df635b4 --- /dev/null +++ b/examples/README.md @@ -0,0 +1 @@ +# Examples diff --git a/pic/animation1.gif b/pic/animation1.gif new file mode 100644 index 0000000..95a6c20 Binary files /dev/null and b/pic/animation1.gif differ diff --git a/pic/animation2.gif b/pic/animation2.gif new file mode 100644 index 0000000..9b33e4d Binary files /dev/null and b/pic/animation2.gif differ diff --git a/pic/dia.gif b/pic/dia.gif deleted file mode 100644 index 0730ac7..0000000 Binary files a/pic/dia.gif and /dev/null differ diff --git a/pic/dia1.png b/pic/dia1.png index acbe704..2aae9c3 100644 Binary files a/pic/dia1.png and b/pic/dia1.png differ diff --git a/pic/dia1.xml b/pic/dia1.xml index 928ee45..b526504 100644 --- a/pic/dia1.xml +++ b/pic/dia1.xml @@ -1 +1 @@ -7Vtbc6M2FP41nmkfkuFu8xhnk+1DO7MzyUy3jzLIWBuMXJATp7++kpAAIYFxAk6ymzwkcDgI6XzfueiSmXu9PXzNwW7zF45hOnOs+DBzv8wcZzEP6G8meC4F3twpBUmO4lJk14I79B8shb4Q7lEMC0WPYJwStFOFEc4yGBFFBvIcP6lqa5yqH92BBGqCuwikuvRvFJONGJUT1PI/IEo28st2EJZPViB6SHK8z8T3Zo675j/l4y2QbVmloNiAGD81RO7NzL3OMSbl1fZwDVNmWWm18r3bjqdVv3OYkSEvzEU3yLMcOoypJcQtzskGJzgD6U0tXfLhQdaARe82ZJvSS5tewgMi35n40hd3/4gnPyAhzwJlsCeYiuq2/8R4J/T07osRsV41BGIwXyHeQpI/U4UcpoCgRxU/IGiQVHrVq98wop9wLEFY2xKACL66nq82UeB9HkHxVtOgpzZEQJ5AojVELxrjqUUcrw6wLbsXvQxng+CyG2AJ6DLaie/qrYSyoAMgV8zH6k9w2S1iPeUfgVksNaIUFAWKSqFQOYURBcnxA7zGKc75CF2L/9Ana5wR0YDt9XGnhE64sAgmHASh5UxCMWoA8NxQ2DHEi24Gur5KHG9h9RKtre8f0fesXn16Ufb4pWwUln0E6V5Y6B4W7PsFNbQTpNTay1VOrxJSgdUgbs1ShvrTBhF4twMctieaYVTmlqSQkdlhZKDMapBkvY6iMDTRxw3c0KWfWSaMmIKvVRRmNxHeMr7y605OPcKcwEMvXcRTp2V3T4b+pzqx2FK2aSYVKTRRTAkbvTFCg2Vy0/N01+e5I1hVekefVb3JjGpPbtQxTOS+pYmc8/Muoj+9Lj+CUd235F04Rb6fqwm/me7L7P+xEz7V1DL+4l0kfHveqhTD/gTe1vdGTuCh7qGvZ5fz67FLzmHfG7380+jljEwvaapGRrAveIGYwke+eLAtFxHOXCpGEfTN9cqYecNpgeEOLAIXI6QNmXqbpXkOUCZrc+s3OjT21/59dOs3C+vjUMRwtVhND4WKxeKcUOhOYIbC+SWhsIMzYhFamkFPync/bVKTCUxJauHYSW2wy/gDXcZ9Y5cZsOYwgctUs42zhK9Qw+LcFp5+iucvjlvYNMUbw8CyLDiXgScwlymGT2au0Rdizm4uQ/UxlbV8E7nKiBmjR9Z4ipKMPwj+3bMdqGVEBwrz+r6Oq+V7LL8o5paK7MFFwTPPFassvd2hu5WVFNwRED2gLLmkr9zRv7RR+q5hlnApX6VjXrWbo7KyX62v5BP3/pbtMjpsU8bCa5aQRukkVeTodCQ0SjuicjiHtOdgxRUYjcVEkWr7y5n/hUpYLVGIqoDdCthTuGZNMS6jCKRXQkxYnbEsqM9QRO550XHhnVJbnJDc3KAVSzTncOR+WtM7vDG840Ms6rbDx1nXH/15ZwAZ7EqOyZXu4XaHc8AHusthjCKCcFbM2M49dQhL9w7S2FuytiBmgYLXhBBEm5nY8e/zKdX5fmanai9Zu77uVbY/lVfp24LjUOYW0TmQQpeKLYIbOvwlW9b8dAYfFqUOBJmM1yaWdfDyk1gMW28AsVwDsXx/BGKZpiJjEOubgVDN+eYn9AbovdCAvCENjbIL5mrIC8zGsbmwpih9dTNvURzzc1ApWMF0WR34Mu2Bt03P70X/DGe1unCoDrqJns+ah8VM+FTxvLW3dMqJqFBp4cJeqC3g9Zr7Qwu4E/ec9MUdDcTGQplYStOKrCFLX33rdZ1IKEtgomtHl8AaTuAbfEDKXntireWCWoU3+MSap3KlCtnjn1gLgw8EtzcQ7kPLbNKMoR4T++jwcuSrJcFTkadhwtzlCZA3TR2mQb4T3aFIth3Lf/dAHo0FIwI5YAnYuLXxSi9WD6IyXXVnZpBP20N9+o1CePBS/K2XhfBTzxR0LPWOdoRUX4K5RZ9VXWX+oDXfst9FUUdv6/9XKNXrfwlxb/4H \ No newline at end of file +7Vtbc6M2FP41nmkfkuFqm8c4m2wf2pmdSabdPsog22owcoWcOP31lYQECAmME3CS3fhhFx2ELuf7zkWXTPzr7eErAbvNHziB6cRzksPE/zLxvPlsyv7lgudCEMy8QrAmKClEbiW4Q//BQhhK4R4lMNfqUYxTina6MMZZBmOqyQAh+EmvtsKp3ukOrKEhuItBakr/QgndyFl500r+G0TrjerZnUbFmyWIH9YE7zPZ38TzV+JXvN4C1ZZTCPINSPBTTeTfTPxrgjEtnraHa5hyzSqtFd/dtrwtx01gRvt8MJPDoM9q6jBhmpBFTOgGr3EG0ptKuhDTg7wBh5U2dJuyR5c9wgOi37n4MpSlv+WbfyClzxJlsKeYiaq2f8d4J+uZw5cz4qOqCeRkvkK8hZQ8swoEpoCiRx0/IGmwLuuVn37DiHXhOZKwriMBkXz1g1BvIsd7EkP5VV2hpzZEAVlDajTEHmrzqUQCrxawHbcTvQxnveBya2BJ6DI2iO96UUGZswnQK25jVRdCdov4SEUnMEtUjTgFeY7iQiirnMKInBL8AK9xiomYoe+IH3uzwhmVDbhBF3cK6KQJS2ciQJC1vFEoxhQAnmsVdhzxvJ2BfqgTJ5g7nURr1g+P1A+czvrsoRjxS9koNfsI0r3U0D3Mef85U7Q3TZm2F0vCnta0BKtG3IqlHPWnDaLwbgcEbE8swujMLUihPLPHycCYVSPJahXHUWSjjz/1I591s1hzYkq+ll6YF2K85XwVz62ceoSEwkMnXeRbr6H3QLn+pyqwuEq2qQcVJbRRTHMbnT7CgGV01Ytw12W5A2hVWUeXVoPRlOqOrtQhVOS/pYq88/MuZr9Okx9Aqf5b8i4aI97P9IBfD/dF9P/YAZ/VNCL+/F0EfHfWyBSj7gDerB8MHMAj00Jfzy7v52OXWsO+N3qFp9HLG5heSlW1iOCK/DCFj2LvYFvsIYwaJOIYhvbkZMgg4TU07/fM+OYDxAgVZ+t5OAEoU4m48wubGv/f/XXwrLyeRR+HIoHL+XJ8KHQs5ueEwmS8HQrvp4TCnZ4Ri8gxFHpScPthI5iKVloEi4aOYL1NJuxpMv4bm0yPDYYRTKZcWpzFfUUGFufW8PjruXB+XMO29dwQClZpwbkUPIK6bD58NHUNvutydnVZso+xtBXayFV4zAQ98sZTtM7Ei+m/e37ctIjZRCGpypVfLb7j8UVTt6rIX1zkIvJc8cwy2B3aW1kqwR0F8QPK1pfskz8BQUC0fsWLd0zM+mBNmSuES9UQ08Cy2TiTFaNs9ElGnsstP2D0+HmMg1c8PA0ySFZRYNUS3hgJqc5oAtnIwVJU4KSWa0RWO1xMwi9MwjOLXOYIvChJkMIVb4ozG8UgvZJiyrOORc4siAFyL1KQi+CUTKO/qbjzacOzGKbiuxZbCYawlQ+xn9t0Jmfdegxnre6ktyl5NlO6h9sdJkBMdEdggmKKcJZP+KE9MwjHtA5aO1ZytiDhfkJkiBDEm4k87O+yKd34fmSjau5W+6FpVW4YjmRV5ongMJS5RWxFpNGlZIvkhgl/wZaVuJghpsWoA0Gm/LWNZS28/CQWxzboQSzfQqwwHIBYtoXJEMT6ZiFUffX5Cb0F+iCyIG8JQ4McgPkG8hKzYXQutSkTYVPNW5Qk4gpUCpYwXZR3vWzH303Vi7Icn+WaVhsO5R03OfJJ/Z6YNZFSJtY4VjrlMlSktXDhzvUW8Gol7KEB3InHTeZWjwFibdtMbqwZSVafjbCu3btWJLQNMTm0oxtiNSMILTagZK+9rNYwQSPD631ZLdC5Urrs4S+rRdMPBHfQE+5DQ21KjZHpE7vo8HLkyw3CU5FnbsI+5BGQty0dxkG+Fd2+SDYNK3z3QB71BQMC2WND2HrQ8Uor1u+g8rr6OU0vm3b72vQbufDpS/F3XubCT71O0LLxO9jtUXML5hZ9ZnWl+qeN9Zb7LpI6Vqz+VKGoXv01iH/zPw== \ No newline at end of file diff --git a/pic/dia2.png b/pic/dia2.png index ef6b952..84570d3 100644 Binary files a/pic/dia2.png and b/pic/dia2.png differ diff --git a/pic/dia2.xml b/pic/dia2.xml index 59bf235..d49e7f8 100644 --- a/pic/dia2.xml +++ b/pic/dia2.xml @@ -1 +1 @@ -7Vtbb+I4FP41SLsPrXIF8lg67ezDrjRSK+3so0lM8DTErGNaOr9+bcdO4tgEKAllO0VqSU5OfDnfd+zjYzPyb1fbrwSsl3/hBGYjz0m2I//LyPOmkzH7zwWvpSCYeKUgJSgpRW4teEA/YSkMpXCDElhoehTjjKK1LoxxnsOYajJACH7R1RY40ytdgxQagocYZKb0b5TQpeyVN67lf0CULlXN7jgqn8xB/JQSvMllfSPPX4hP+XgFVFlOKSiWIMEvDZF/N/JvCca0vFptb2HGLausVr53v+Np1W4Cc3rICxPZDPqqug4TZgl5iwld4hTnILurpTPRPcgLcNjdkq4ydumyS7hF9DsXX4fy7h/55Aek9FWiDDYUM1Fd9p8Yr6We2XzZI96qhkB25ivEK0jJK1MgMAMUPev4AUmDtNKrXv2GEavCcyRhXUcCIvnqB6FeRIE3JIbyraZBjy2IApJCahTELhr9qUUCrx1gO24nejnOD4LLbYAloctZI77rtwrKgnWA3nAfq6sQsnvEWyoqgXmiNOIMFAWKS6FUOYYRBSX4Cd7iDBPRQ98RH/ZkgXMqC3CDLu6U0EkXloOJAEE57yAUYwYArw2FNUe82M1AP9SJE0ydTqK19cM9+kG3PrsoW/xWNkrLPoNsIy30CAtef8EM7Y0zZu3ZnLCrlFZgNYhbs5Sj/rJEFD6sgYDthc0wOnNLUqiR2eNkYMxqkGSxiOMostHHH/uRz6qZpZyYkq/VKMxvYrzifBXXOzn1DAmF2066yKdea0AI1ND/Uk8srpItm5OKEtoopg0bnWOEAcvgphfTXZfn9mBV5R1dVg0GM6p7fqPG7NPJ5z6M6r+nUb2PaVT/PZkaDREhTPQQoRkglPHC/ztEYJpGjDC9iBDBnbRiy6h7ym/rBz1P+ZHpoaezy7toBnXR/s3s8oLLpFd4HL28PRGooe900HH/2wMurAJjKnKvRCybwWeR51iV+Y4zR7VxDEN7aNXnhOW1WOAfGK9Oe5iv1JzfXEUQgHK1jHB+Y13j3+7vvVu/uQbYD0UC59P58FDoWEzPCYXpBHYovHeG4oC13gBQuOMzYhE5hkGPmmg/VKw2dCrQ9+zT1NGpwHC6p6D+ZiyVN9/rrP4vOW5WC6yzDJyRgcW5h8PhV7UGty0Wtq1q+zCwCkh+KQPb5pvBDNx7fqvbwAOYyxIpDWWt0EbHcoxN0DMvPENpLh6M/93wjb1ZzDoKSX1fj8Tle3wubMvmSvBAQfyE8vSaFf3AvhkiLB62rE+u1ausB/N2cUxmrYUc1ZZ7vrHKQkD2hxd8eumlSqYoLLdjemKUoDq/CCzQTzAXCpxicqXLtMPZKPzCJDwmKWR0wW8lJBlc8KI4z1AMshsppjxemRWMz8y+jyJ4uQqOiVGOmKr8ccvPDeJWK90mc4M+mDt8HnsA1z5rAjWc7HTukssNWylv5g+uSrLdcNf01lvT1R/hao0JEB1dE5igmCKcFyN+WIE5hGN6B21spzkrkHC3FxEeBPFyJA85dPmU7nwf2anaOXc/NL3KDYfyKnMntB/K3CO2ltLoUrFFcsOEv2TLQhxIEd1i1IEgV+O1jWU7ePlJLI5tcACxfAuxwrAHYtkWFn0Q65uFUM3V4yf0FuiDyIK8ZRrqZRvPN5CXmPVjc2lNGZaaZl6hJBFHvzIwh9msOuNm2/Zvm17cy/ZZjqftwqE62ydbPmqej7PhU43nrc2xYw6BRVoJV+5ULwEvFsIfTkrVRGaqxgCxkXCTKTkjyBpss0rbmJJN25tyazhBaPEBJTv1kF7LBY0I7+BDeoHOlWrI7j8zF40vG+690G5bJlIJn4k5/nVB/3aUjUX6oSgbWcB2QT2ibFsmDIPyyUi2nSi8eCD3+n2PQB6QvLVuh5zosfo5W66r7+YcNFyrfesLHa7fvJHSXs21V2k74D/2/MOO7ZreDsia2ZZ79BnAVeYft1B2LyJ+Y7f1rzFK9foHL/7dfw== \ No newline at end of file +7VvbbuM2EP0aA+1DAl1t6zHOJtuHFlggQbt9pCVaZiOLLkUnzn59SYqURJGW7Vhy3GwC7EYcjXiZc4YzvGTk3662XwlYL//ACcxGnpNsR/6XkedNJ2P2Pxe8loLAk4KUoKQUubXgAf2ApTCUwg1KYKHpUYwzita6MMZ5DmOqyQAh+EVXW+BMb3QNUmgIHmKQmdK/UEKXclRqEFz+G0TpUrXsjqPyzRzETynBm1y2N/L8hfgpX6+AqsspBcUSJPilIfLvRv4twZiWT6vtLcy4ZZXVyu/ud7yt+k1gTg/5YCK7QV/V0GHCLCGLmNAlTnEOsrtaOhPDg7wCh5WWdJWxR5c9wi2i37n4OpSlv+WbfyClrxJlsKGYieq6f8d4LfXM7ssR8V41BHIwXyFeQUpemQKBGaDoWccPSBqklV716TeMWBOeIwnrOhIQyVc/CPUqCrwhMZRfNQ16bEUUkBRSoyL20BhPLRJ47QDbcTvRy3F+EFxuAywJXc468V0vKigLNgB6w32sbkLI7hHvqWgE5onSiDNQFCguhVLlGEYUlOAneIszTMQIfUf8sDcLnFNZgRt0caeETrqwnEwECMp5B6EYMwB4bSisOeLFbgb6oU6cYOp0Eq2tH+7RD7r12UPZ47eyUVr2GWQbaaFHWPD2C2Zob5wxa8/mhD2ltAKrQdyapRz1lyWi8GENBGwvLMLozC1JoWZmj5OBMatBksUijqPIRh9/7Ec+a2aWcmJKvlazMC/EeMX5Kp53cuoZEgq3nXSRb73WhBCoqf+lDiyuki2bQUUJbRTTpo3OOcKAZXDTi3DX5bk9WFV5R5dVg8GM6p7fqDH76eRzH0b139Oo3sc0qv+eTI2GyBAmeorQTBDKfOH/nSIwTSNHmF5EiuBOWrll1B3y2/pBzyE/Mj30dHZ5F82gLtq/mV1ecJn0Co+jl7cnAzX0nQ467v96wIVVYIQiV6SyGXwW2xyrcrtj0OgUxzC051F9RievBbl/YHI67SE4qQDfXDIQgHK1ZnB+YUPjv91fe19ANBP+/VAkcD6dDw+FjsX0nFCYjLdD4b0zFAcs7AaAwh2fEYvIMQx6VFT9UInZ0Pt+vmePSUfv+4XTPRX1F57UJvleZ/V/ynmzWk2dZeKMDCzOPR0Ov4Q1uG2xsG0J24eBVULyUxnYFm8GM3Dvm1ndBh7AXJZMaShrhTY6lnNsgp555RlKc/Fi/O+Gn+LNYjZQSOpyPROX3/FY2JbNleCBgvgJ5ek1q/pPQBAQMeeGFx+YmAHE0mNzbXKtKmLjmbcrZzJrm+Sont3zM1WWELJ/eMGDTS9NMkVhxx3BihGE6mwjsEA/wFwocMLJRS7TDmej8AuT8AylkLkGL0qAMrjgVXHWoRhkN1JMefYyKxi7mXkfRSpzFRyTsRxOY3c6bnm9QWPftfA46IPHw29hD+DoZ907DSc7Xb3kcsNWyrf5i6uSbDfcM7311nT8R7haYwLEQNcEJiimCOfFiN9TYA7hmN5BGydpzgok3OtFvgdBvBzJ+w1dPqU730d2qvZ2ux+aXuWG4UBeZR6C9kOZe8RWVhpdKrZIbpjwl2xZiLsoYliMOhDkar62sWwHLz+JxbENDiCWbyFWGPZALNsyow9ifbMQqrmW/ITeAn0QWZC3hKFeTvB8A3mJWT82l9aUSapp5hVKEnHrKwNzmM2q6222E/+26UVZ9s9yM20XDtW1PtnzUfNqnDWRUi7WOhc75v5XpNVw5U71GvBiIfzhpI2byNy4MUBsbL/JDTojyRrsnEo7k5Jd27sB13CC0OIDSnbq/byWCxoZ3sH38wKdK9WU3f8+XTS+bLj3QrttmUht/0zM+a8L+rejbCzZD0XZ2BNsV9QjyrZlwjAon4xk24nCiwdyr9/3COQBW7nWw5ETPVa/Yst19bOdg6ZrdWR9odP1m49V2qu59iptB/zHXn3YcXjT291Yc7flHn0mcJX5xy2U3YvI31ix/kOMUr3+Wxf/7j8= \ No newline at end of file diff --git a/pic/dia3.png b/pic/dia3.png index e7dafaf..9c113bb 100644 Binary files a/pic/dia3.png and b/pic/dia3.png differ diff --git a/pic/dia3.xml b/pic/dia3.xml index 07d426c..a7f3fd2 100644 --- a/pic/dia3.xml +++ b/pic/dia3.xml @@ -1 +1 @@ -7VvbbuM2EP0aA+1DAl1t6zHOJtuHFgiQBbp9pCXKZiOLLkUnzn59SYrUjbRsxbRzDbAbaTQixTlnhpwhM/KvV9vvBKyXf+EEZiPPSbYj/9vI86aTMfufC55LQTDxSsGCoKQUubXgHv2CpTCUwg1KYNHSoxhnFK3bwhjnOYxpSwYIwU9ttRRn7U7XYAE1wX0MMl36N0roUo7KG9fyPyBaLFXP7jgqn8xB/LAgeJPL/kaen4qf8vEKqLacUlAsQYKfGiL/ZuRfE4xpebXaXsOMW1ZZrXzvdsfT6rsJzOkhL0zkZ9BnNXSYMEvIW0zoEi9wDrKbWjoTw4O8AYfdLekqY5cuu4RbRH9y8WUo7/6RT/6FlD5LlMGGYiaq2/4T47XU0z9fjoh/VUMgB/Md4hWk5JkpEJgBih7b+AFJg0WlV716hxHrwnMkYV1HAiL56gdhu4kCb0gM5VtNgw5tiAKygFRriF00xlOLBF47wHbcXvRynB8El9sAS0KXs4/42b5VUBZsAPSK+1jdhZDdIv6lohOYJ0ojzkBRoLgUSpUhjCgowQ/wGmeYiBH6jvhhT1KcU9mAG/Rxp4ROurAMJgIEZcaTUIwZADw3FNYc8WI3A/2wTZxg6vQSrasf7tEPBuqHTq8+uyhH+FL2SiQeQbaRFv0BC95/wYDxxhlDZzYn7GpBK3AbRK9ZzVnytEQU3q+BgPmJzUhtppckUpHc4+RhTGyQKk3jOIpMdPPHfuSzbmYLTmTJ7ypq85sYrzi/xfVODj5CQuG2l17yqdexe6Cmiqd6InKVbNmchJTQRMlWmOmNKdPjYkoVOZpxowwj7ztyqDDRDB2h9dnpcJw09zmDi8Rxr4tYYL+Ken3sD05GfveDGtV/TaN6H9Oo/msyNTrFym/SXvp9uAAe6AF8ajt+v2jp5046OUPUvzTr6geWl2aR7qHHs8t70wzqo/2L2aUi3VujVziMXt6eTEHTd4bpu14Pffe/Pe7EU4sJdqBNXe6FyFEy+CjqXauy7nXmbCWOYZimp57gvA5r/APzkKmF+U15TjM7JADlKj10fmND47/d361bv5nb2UgcrUDRxmJ6Tih0JzBD4b0yFHEC59P5uaGows85sIgczaCfNzk/dUnY98zT1OCScDjd05C9GUvtn+x1Vv9TOmuVkJ0lcEYaFueemU6fBXe5bbKwKQu2YWC1IPlUBjbNNyczsPV62DswsGFtdSr7hiYCl1E5QY+88QwtcvFg/N+GbwnPYjZQSOr7OnaX7/HZsyubK8E9BfEDyheXrOl79pthyFbQhozmUr3KRjDvNsdkxl7IoG+55VvybB5i/3BaXljokikKy+2Y0BglaJuRBBboF5gLBU5KmUsz7XA2Cr8xCV/FFHI9wm8lJBlMeVOcZygG2ZUUU77CmRXMA5h9f4jlzkUwZFUzYHLzx53IoBHXU5vPTeYGNpj7QSvl3WBw1qJuONkZDkr2N6yr/J8/uCjpecWd2Vtv9eDwA67WmAAx0DWBCYopwnkx4gdjmAs5uj/RxlasswIJDxRiFQlBvBzJAzV9Xth214/sht19AD/U/dANT+WH+i66HcrcIpavtehSsUVyQ4e/ZEsqDj+JYTHqQJCrCG9i2Q5efhGLYxscQCzfQKwwtEAsU/Jig1h3BkI1M9Qv6A3QB5EBecM0ZGVr0deQl5jZsbm0plzI6mZeoSQRxwwzMIfZrDpP2VoLiBOVuunFvfw+w1HIXThU50jll4+aZzFN+FTxvLNhN+TAYdRq4cKdtlvAaSr84ahyUKSXgzQQG0U9WfbTlmUn20DbW8JrED408F3Jjj382XG3qmw0tNKn1Zu6Ddmr9EXjdw6tsr3fdiVvose6PuiPQLmbwr8Y5W5DFlE2pQSnQfloJDvmdcM3D+Rev7cI5AHFYOP2youwbO8A7TrNXT27gwSxsfDpuP8Ua+vYdPC2Y3g3vTqUFN10zjvs/P7QQxk76ubWTlfrBZpb9LWCq8w/7qDsvokFHLut//SnVK//usq/+R8= \ No newline at end of file +7VvbcuMoEP0aV+0+JKWrbT3GmWT2YbcqVUntzjxiCdlsZOFFOHHm6xcQyJLAsh0jx7mkaiaiQSD6nG66gQz868X6OwHL+V84gdnAc5L1wP828LzxaMj+54KXUhB4UjAjKClF7kZwj37BUhhK4QolsGi0oxhnFC2bwhjnOYxpQwYIwc/NZinOmoMuwQxqgvsYZLr0H5TQuZyVmgSX/wHRbK5GdodRWTMF8eOM4FUuxxt4fip+yuoFUH05paCYgwQ/10T+zcC/JhjT8mmxvoYZ16zSWvne7Zba6rsJzOk+L4zkZ9AXNXWYME3IIiZ0jmc4B9nNRjoR04O8A4eV5nSRsUeXPcI1oj+4+DKUpZ+y5l9I6YtEGawoZqJN339ivJTt9M+XM+JfVRPIyXyHeAEpeWENCMwARU9N/ICkwaxqV716hxEbwnMkYV1HAiL56gdhs4sCr0gM5Vt1hR7aEQVkBqnWEXuozWcjEnhtAdtxO9HLcb4XXG4NLAldzj7iR7OooCzYBOgVt7HNEEJ2i/iXikFgnqgWcQaKAsWlUDY5hBEFJfgRXuMMEzFD3xE/rCbFOZUduEEXd0ropAlLZyJAUGrshWJMAeCl1mDJES+2M9APm8QJxk4n0drtwx3tgwPbh05ne/ZQzvC17JVIPIFsJTX6AAs+fsGA8YYZQ2cyJexpRitwa0TfsJqz5HmOKLxfAgHzM1uRmkwvSaQ8ucfJw5hYI1WaxnEUmejmD/3IZ8NMZpzIkt+V1+aFGC84v8XzVg4+QULhupNestZr6T1QS8XzZiFylWxeX4SU0ETJhpvp9Cnj43xK5TnqfqN0I+/bcyg3UXcdofXVaX+cNPM5gYnEcaeJWGC/8npd7A96I7/7QZXqv6VSvY+pVP8tmRr1EfmNmqHfh3Pgge7Ax7b996tCP3fUyhmi7tCs3T6wHJpFuoUezy7vrBnURftXs0t5unOjV3gYvbwdmYLW3jmsvet10Hf328OWP7WYYAfa0uWKFCWDT2K7a1Fue/W6msUxDNO079XMa1HE3zPpGFtYzJSZ1FNBAlCuckHnNzY1/tv93XpiWE/kbGSJVqBoYjE+JRQ6481QeG8MRZzA6Xh6aigqX3MKLCJHU+jnzcT73v/1PfOadPD+bzje0ZG95Ukdluw0Vv9TGmuVfZ3EcUYaFqdemfpPedvcNmnYlPLaULAKSD6Vgk3rTW8Ktr759Q4UbIit+tJvaCJw6ZUT9MQ7z9AsFxXD/1b8/HcSs4lCsilvfHf5Hl8927KpEtxTED+ifHbJuv4bEATEKnXFi/dMzCBlAbWezVyqjth8pu3Omcw4Jjnoy275aTxbldg/nJYPFoZkDYUetyxvjCC0yU8CC/QLTEUDTlGZRrPW4WQQfmMSHtMUMjrhRQlQBlPeFWcdikF2JcWUxzuTgtkDU++DCH4ugkNinP1p7I6HLT+h0dh3DTwObPD4g26St13DSfdzw9FW51Cyv6Zd5Q14xUVJzytuy95yrbuKB7hYYgLERJcEJiimCOfFgN+JYSbk6PZEa6ewzgIk3E+ImBKCeD6Qd2m6rLBprh/ZDNtHAH6o26Ebhj3ZoX6Abocyt4hlbw26VGyR3NDhL9mSintPYlqMOhDkysObWLaFl1/E4tgGexDLNxArDC0Qy5TK2CDWnYFQ9Xz1C3oD9EFkQN6wDFk5VfQ15CVmdnQutSnDWl3NC5Qk4oZhBqYwm1RXKRuxgLhMqatelOX3GW5BbsOhukIqv3xQv4ZpDL2UibXO6g65axg1erhwx80ecJoKezhqcyjSN4c0EGtbfHITUAvLejs727mhVyN8aOC7kh1777NlbtUm0qH7ftruU7sje/t+0fCdQ6t07zdNyRvpvq4L+iNQbif0r0a53ZFFlE0pQT8oH41kS71uePZA7rR7i0DusTVsPGx5FZbN86BtF7mrujtIEJsLX467L7A2bkwH5+3D2+nVvqRop3Peflf3D72PsWUX3drFan2D5hZ9RXCV+octlN2zCOBYcfNXP2XzzR9W+Tf/Aw== \ No newline at end of file diff --git a/pic/dia4.png b/pic/dia4.png new file mode 100644 index 0000000..2f79f82 Binary files /dev/null and b/pic/dia4.png differ diff --git a/pic/dia4.xml b/pic/dia4.xml new file mode 100644 index 0000000..1841901 --- /dev/null +++ b/pic/dia4.xml @@ -0,0 +1 @@ +7Vrdc6M2EP9rPNM+JMOnYx7jXHJ9aGduJjft3aMMMlaDkSvkxOlf35WQDEJAcA6cy/U8Ewcty0ra336x8sy/2R4+MrTb/EETnM08JznM/A8zz1tczeFbEJ5LQuApQspIUpLcinBP/sUlMVTEPUlwYfBxSjNOdiYxpnmOY27QEGP0yWRb08ycdIdSbBHuY5TZ1L9IwjdqV3oTgv4bJulGz+zOo/LOCsUPKaP7XM038/y1/JS3t0jLckpCsUEJfaqR/NuZf8Mo5eXV9nCDM6FZrbXyubuOu8d1M5zzIQ9cqWXwZ711nIAm1JAyvqEpzVF2W1GXcntYCHBgtOHbDC5duMQHwr8I8mWoRl/Vnb8x588KZbTnFEiV7N8p3Sk+e/lqR2JVNYLazEdMt5izZ2BgOEOcPJr4IWUG6ZHv+OgnSmAKz1EG6zoKEGWvfhCaIgq6ZzFWT9UVeqogjliKuSUILmr7qUgSrw6wHbcXvZzmg+Bya2Ap6HJYxBdzqKEsYAP8WvhYNYWk3RGxUjkJzhPNEWeoKEhcEhXLKRZRcEYf8A3NKJM79B35gTtrmnMlwA36bKeETrnwJOYEm0XPNYadQLfotjY/NI0kWDi9RtXkD1/gD5xefrgoVzzw6cCZyoQVHI8o2ytVf8aFWEqB4XueAZDLFYOrlB8Rrll7ZdrCVJ42hOP7HZJYP0FaMs29tCQdzj1hQWCONcvC8tNmc/7cj3yYZpkKa1ZGfgzdYhDTrTByed1piI+YcXzotTt112tAEOh88VRlI1fTNvVM5DWwqtuqAVQPKpE3QVzxzMBSDytllHnfgQU4yzJF+oZCMfwuoo171UhJYX/0aPJ7fdHjNXkrsLzelU6f4UdZRW7LanJSZ49jHEJV1ufso/ixqXl/oBsvRvBiz7ODK0Mk19HV+QW2Jv67v44eauuh8WUoErxarKaHwsRicU4obItvh8L7X0Lhzs+IReRYCj0puf2wGUxnKyODRWNnsMEuEw50Gf+NXWa9juMoOrfLeME5w1dkYXFuDcfx1BrW70h9Gg4mUrAuC86l4AnU1RbDJ1OX++7V1VJ9TKWtsM24yoiZkEchPCNpLm/M/9mLxuMyho1iVo2ruFo+J/KLoW7NKG5cFDLzXIvKMtgduqWsNOGeo/iB5OklPPInYgRJ6UsxvAcyzAGi7DeESy0INLBqCgdaucrGnGzyveCd4IY/uoavYJRFAqPEqiO9gRFy06IZhpWjlWQQRq3eEYE7XM7CD0ARlUWhagQxVEaQ4bUQJSybxCi7VmQuqo5lAR4EgHyWJchFcEqlMdxV3MW8EVksV/HdFl9pdqle5SujR5ZzBJO27lBbNBmlORTaPbuTXclrc6VPDCck5oSKGm8NKR5K0Voz0On1DtONfmT3CPWBgu4pLFy7EvRb0B8F/LZKcCrw6+X+T+hntUa89vvofMhHvoW8wmwcnSttqsrDVvOWJIk8fczQCmfL4zGr8Z4gD1pt1cuxWl/LCWkXDsfjZbXyWf2ItjVzhQ3XjHoU39F7jQwJF+7ClEDXa+kPDeBOa8BG9ru1BWKtT6E6GVZWG9J56GuXdCJhdCDU0l7sQNScIGzxAU371nPihgtaKXXwOXEjjLv+ZOfE0fwdwR0MhPvQUJtWY2THxD5zeD3yx47MqchDmGhf8gTIX50N+U50hyLZdKzwuwfyxVgwIpADOnCtneVv9GLz5x+C12yMD/Jpd6hPv1EIn78Wf+d1IfzU89uOTttYx7EL+533jvys6o7qn5vqP/Ye3raog2H1K8GSvfohpn/7Hw== \ No newline at end of file diff --git a/pic/dia5.png b/pic/dia5.png new file mode 100644 index 0000000..e0408ee Binary files /dev/null and b/pic/dia5.png differ diff --git a/pic/dia5.xml b/pic/dia5.xml new file mode 100644 index 0000000..03b334c --- /dev/null +++ b/pic/dia5.xml @@ -0,0 +1 @@ +7Vpbc5s4FP41nuk+JMPVNo9xmnQfdmc6485u+yiDwNpg5Ar5kv76lYRkEAKME/B6tvVMHHQQQjrfdy468sR93Bw/EbBd/4kjmE4cKzpO3I8Tx5nPpuybC14LgedIQUJQVIjsUrBEP2Ah9KVwhyKYa/0oxilFW10Y4iyDIdVkgBB80LvFONVfugUJNATLEKSm9G8U0bVclVoEl/8OUbJWb7anQXFnBcKXhOBdJt83cdxYfIrbG6DGsgpBvgYRPlRE7tPEfSQY0+Jqc3yEKdes0lrx3HPL3dO8Ccxonwdmchr0VS0dRkwTsokJXeMEZyB9KqULsTzIB7BYa003Kbu02SU8IvqVi+992fom7/wDKX2VKIMdxUxUjv0HxlvZz5y+XBGfVUUgF/MJ4g2k5JV1IDAFFO11/ICkQXLqd3r0M0bsFY4lCWtbEhDJV9fz9SFyvCMhlE9VFXrpQBSQBFJjIHZRWU8pEni1gG3ZnehlOOsFl10BS0KXsUl81ZsKypwtgD5wGytfIWTPiM9UvARmkeoRpiDPUVgIZZdLGJFTgl/gI04xESt0LfFhd2KcUTmA7XVxp4BOmvAodGKLBa+VDluObt7ONtfXSeLNrU5S1fv7Z/p73f3ZRTHjnk971lgUlnDsQbqTqv4Ccz6VHLLvacqAXKwIu0roCeEK20tqc6oc1ojC5RYIrA8sLOl0L5ik3LnDGcToWGEWFJ8mzrlTN3DZaxYJZ7Mk+cl180aIN5zk4rqViHtIKDx28k7edWpexFPx4lBGI1vJ1tVI5NSwqnJVA6oDlcAZwa84/i07j9LjVf1d4f56ORY2SpGmCNuQKHo34W3sWS0k+d3eo97fOeNtjP7WJd7GeHrEgOkZ7sYW3iaFe5G+boo0dlQvE4bQZ+lgl5cZxIHokLs9/cd8APfhOKZXJwBlyq1bH9jS+H/7t8F9fNUnn4cigqv5anwodCzm14TCZHwzFM5/DEUch2EQXBsKe3pFLALLUOhFUbUjQt1sWG3FbOz9nOs0x6SL93MqdW4daLjwpIofZ43V/Sn9puNd03EGBhbXdodhOLaGDW43aNgbScEqIfmpFNwUb0ZTsH1dBY+groZMaSxt+U10LHxshPZ88BQlmbgx/b7j1dlFyBYKSdkuPXHxHI+FddlKCZYUhC8oS+7Z0H8BgoCIOQveXDIxA4ilx+be5F4NxNazqg/OZI3vJBfODG55Qsj+cMy+vEFeyToKPbYEK0YQqrONwBz9ACvRgRNObnJZb38x8T8yCc9Qcplr8KYEKIUxH4qzDoUgfZBiyrOXRc7YzdT7RaQyd94lGUt/Gtvzac3qDRq7dgOP62W2N/F4cKu/hqE3lbeaLH2Q6pZvFh11w6joStk2v3FXkO2BW6azPZqG/5nACIUUYZ6xxSyeOBatVDOtTuvQzej/bB6+KvCo2sTcNvM6twH9QcBvyuvGAr+avP+CflI5SVB2H1wP+cA1kJeYDaNzqU2ZFZhq3qAoEsenKVjBdHE6J9aSUnFSbKpetOX8Go5423A4nY/LmU+qZ8yNkcuvmWbQofiWcm6gjXBnz/URcBwLe3jXTjkwd8oGiJV6h6yIGFFttIMB7RBATu1sxaNiBH6DDSjZew+6ayZohNTeB901N267o9Xtg+ltw30W2mNNRWq/PTP9Xxf0b0fZ2CP1RdkowtQHGhDl2dVQfjeSdSPybx7Is3Y/IJA9ameN1eh3Wqz+WxXeVy+m93LX6ozwRt31m+vYtR2X2/M3HZeeNbdUyxuPgt/yexFze/uMfiVwJ/VPayjbN5G/sWb5i8aie/mjUffpXw== \ No newline at end of file diff --git a/pic/dia6.png b/pic/dia6.png new file mode 100644 index 0000000..6b667e5 Binary files /dev/null and b/pic/dia6.png differ diff --git a/pic/dia6.xml b/pic/dia6.xml new file mode 100644 index 0000000..705c0ac --- /dev/null +++ b/pic/dia6.xml @@ -0,0 +1 @@ +7Vpdc6M2FP01nmkfkgEBjnmMs8n2oZ3Zmey020cZZKwGI1fIibO/vpKQQCDAJgHXbdYzm0WXi5DuOfdDEjPvbnv4TOFu8xuJUToDTnyYeZ9mACxu5vyvELwWAh8oQUJxXIjcSvCIv6NCGCjhHscor+kxQlKGd3VhRLIMRawmg5SSl7ramqT1l+5ggizBYwRTW/oHjtlGzUpPQsh/QTjZ6De787C4s4LRU0LJPlPvmwFvLX/F7S3UfTmFIN/AmLwYIu9+5t1RQlhxtT3coVRYVluteO6h4245booydsoDN2oY7FVPHcXcEqpJKNuQhGQwva+kSzk9JDpweGvDtim/dPklOmD2TYivA9X6U935CzH2qlCGe0a4qOr7V0J2Ss8evpqRGJUhUJP5jMgWMfrKFShKIcPPdfygokFS6pWPfiGYvwI4irCuowBRfPX8oN5FTvY0Quop06BDO2KQJohZHfELYz6VSOLVAbbj9qKXkewkuFwDLAVdxgfxrd7UUOZ8AuxW+Fj1Cil7wGKk8iUoi7VGlMI8x1EhVCpDGJEzSp7QHUkJlTP0HPnjd9YkY6oD1+/jTgGdcuFJ6MQnC18NhZ1AN+9mmxfUSeIvnF5SNfWDI/r+QP3A6dXnF8UMT3zad6aivILvGaZ7Bc1XlIuh5Ij/nacc+OWK8quElYwwvKNyBUGtlw1m6HEHJTdeeBqru0fBPB3+gWAcp6/BRCR/bRz15l7o8dcsE8F+5RRlqBeNiGyFU8jrTuI+I8rQoZen6i5oQODr/PJSZS9XyzZm5gINrExu14DqQSUEE8QhEFxysKkipBkfi3B5UiDivRRljfQNhSK4iOjk3jRSWNAfPZr64Ei0sfSdYfouGBKdrKfnk0UnjakRnlwZnVL0LMvjbVEmTxqVoggFvNzsi0qjBJw6RbwT481ihHCj3cTMAhTiTKcB5yc+NfG/+/PoOcGM4UehWK+jKAynh6KOxeKcUNiMb4cC/MtQRDFaLVbnhqKMNefAInQsgw7Kwj0Z7WLTcCdmU68XPdCekwavF3Wp3dnReOlJb64cdVbvQzor8M8ZOEMLi3Nnpiia2sJNbrdZ2J/IwLog+VAGbss3kxnY/YAGbqmtprJv0EbgIirH+Fl0nuIkkzfmf+/FfvEy4hNFtGpXsbt4TmTPpmylBY8MRk84S655179DiqHMUkvRfORiDikvqO3VzLXuiM9n1eycy1rfSQeODO1EVuL/yJr/8Ud5JVeUduxIb5wgrM5PinL8Ha6kgqCoWkZz7WA5Cz5xiahpclWdiKYCKEVr0ZVgHY5geqvETNQ7y5z7AzfvV1n8XPlDapzTaewu5o04YdHYc1t43NzIexOPR48Tx0ODPHDpqyMnCA1tW25tsWGUHbfA3gitu5JhXR0NxI2rgp63wpfB7mCHii8UxThimIiqcM3tBhxm7LA6vf5Ud7z/s0MF+lRH738sXLt29FrQHwX8ttpxKvDNBcIP6GfG6Yb2+/B8yIeehbzCbBybK2uqOsI28xbHsTwCTuEKpcvyrLst+DZNL9tqfC3H1F04lGf8auQz85y8NdcFDdcMewzfsWUc1nq4chf1Hsh6Lf3hXavx0F6NWyAaeypq18XKg5MdVhzdQTEIH7TwXcveezDfcLdy1T50o8Va7jc7Gm+jJZz/x6HVtvfqrgRu7FjXB/07UG6uoN6McrOjEVG+ORvK70ayYV43uHggj/r9iECesBfXurv9JizrG/BdX9qU974givlcRDru/1jAPEjWB5GXGsOb68pTSdFYcnngtG+rhh6Ad2xbtp43v+UjFntF/IB/VHCl+ecNlN2LKOB4s/oss1Cvvnz17v8B \ No newline at end of file diff --git a/pic/dia7.png b/pic/dia7.png new file mode 100644 index 0000000..4f33879 Binary files /dev/null and b/pic/dia7.png differ diff --git a/pic/dia7.xml b/pic/dia7.xml new file mode 100644 index 0000000..cfde42d --- /dev/null +++ b/pic/dia7.xml @@ -0,0 +1 @@ +7VpLc6M4EP41rto9JMXTNsc4k8wedqtS5andmaMMMtYGI6+QH5lfvxJIgCSMsQMuz2RycFCr0aO/r7v1YOQ+rg+fCdis/sIRTEaOFR1G7qeR40wnY/bLBW+FwHOEICYoKkR2JZij77AQ+kK4RRHMFD2KcULRRhWGOE1hSBUZIATvVbUlTtRONyCGhmAegsSU/oMiuhKzkpPg8j8gileyZ3scFDULEL7GBG9T0d/IcZf5X1G9BrItqxBkKxDhfU3kPo3cR4IxLZ7Wh0eYcMtKqxXvPR+pLcdNYEq7vDARw6BvcuowYpYQRUzoCsc4BclTJZ3l04O8AYuVVnSdsEebPcIDol+5+N4XpW+i5l9I6ZtAGWwpZqKq7T8x3gg9c/hiRnxUNYGYzGeI15CSN6ZAYAIo2qn4AUGDuNQrX33BiHXhWIKwtiUAEXx1PV9tIsNbEkLxVt2g5zZEAYkhNRpiD7X5VKIcryNgW3YreilOO8Fl18AS0KVsEF/VooQyYxOgD9zHqi5y2TPiI807gWkkNcIEZBkKC6FQOYcRGSX4FT7iBJN8hq6V/7GaJU6paMD22rhTQCdceBA6scmCt5rChqObHWeb66sk8aZWK6l0ff+Evnemvm+16rOHYoYd3/asoSgv4NuBZCug+QIzPpQMst9xwoCfLQh7imnJiJp3VK7AqbVfIQrnG5BzY8/SmOoeBfNk+Hc44xh9a0xcLsMwCJo46o7dwGXdzGLOfuEUZajnhRCvuVPkz0eJu4OEwkMrT0Wto0Hgyfyyr7KXLWWreuZyNKzq3FaAag1E0/cFojLc1INNEXtuNtwcj57nBKLCNYTfXCnX+U5z9DmV6y7w12CIDDXxf1TOdGIG0zSocRtpy55oa5ugPa3o+l5bWrlkAeQZ6cDOs0ECd/l2ZF1sSwbNAmEIfba8b8sCvQR41fJux/iuu/Yl4Z2r6lmXAJTKtGv9xqbG/9u/956D6znzNBQRXEwXw0OhYjG9JhQm45uhcD4kFPb4mlj4HbFwPyQWjndNLAIDi2tamO8EwnBoC8tdWZuFvYEMLPPNhzJwUzgZzMD2BzRwQ+ocyr5+E4GLqByhHW88QXGaV4z/2/Lj11nIJgpJVa5id/EeX93rsoUUzCkIX1Ea37Om/wYEgXxhPOPFORMzSNl6yVys3suG2HwWeuNM1tgnOXNkcMO6YysJCy+Lhx66ZIq5HY+kN0YQqvKTwAx9B4tcgVNUbD6Ytj8b+Z+YhO+5MrF74kUBUAKXvCnOOhSC5EGIKd+PzTLmD8y8X/LN2Z13zh6sO43t6ViLEwaNXbuBx/q52EU87j1O3GZoaDrBaooNvRxg+ea5oupKNevKaMAr7gp6PnBfdjYHM1S8EBihkCLMV4VLZje2ra8dWFqt/qQ63s/sUL68JJHb26ltrh3dBvR7Ab9p7TgU+PUNwi/oR7XLAun3wfWQD1wDeYFZPzYX1hTrCNPMaxRF+Y1qAhYwmZVXx0rwzS+PTdPnZTG+hlvfYziUV+Zi5KP6tXNjrvM11wxaDH/kGDBQWrizp2oLeLnM/UED7syTZnM3boBYO/MVp8JGHux08t9y9HwUiZNHtzXC+w18l7L33nNr7lbu2s+95za2+3pD/V36BeMfHFppe1d1JWdixro26N+BcscbntMo6w31iPLkaii/G0nNvLZ/80Ce9PsegexwFtd4+3YRlurd4bEPV8q6F0gQmwtPx+137/XbN3nP9LPFcG3L5TrdPlU699pQP7a0+r0FnJo74mf0awVXmn+soWzf5gLOts6LG8Omec3VG6C5Fe83DkAuze1GQz1+qSg5d7Xkfgl+zcn9eou0S8P4SUZcDCQrVp8iF+rV197u0/8= \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eb62c16 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools >= 42.0.0"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 814dcc9..ac91fae 100644 --- a/setup.py +++ b/setup.py @@ -2,17 +2,53 @@ from setuptools import setup +long_desc = ''' +Python package for stacking (stacked generalization) featuring lightweight functional API and fully compatible scikit-learn API. +Convenient way to automate OOF computation, prediction and bagging using any number of models. +All details, FAQ, and tutorials: https://github.com/vecxoz/vecstack +''' + setup(name='vecstack', - version='0.1', + version='0.5.2', description='Python package for stacking (machine learning technique)', + long_description=long_desc, + classifiers=[ + 'License :: OSI Approved :: MIT License', + 'Operating System :: MacOS', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: Unix', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + ], + keywords=['stacking', 'blending', 'bagging', 'ensemble', 'ensembling', 'machine learning'], url='https://github.com/vecxoz/vecstack', - author='vecxoz', + author='Igor Ivanov', author_email='vecxoz@gmail.com', license='MIT', packages=['vecstack'], install_requires=[ 'numpy', 'scipy', - 'scikit-learn' + 'scikit-learn>=1.6.0' ], + extras_require={ + 'test': [ + 'pytest', + 'pytest-cov', + 'pandas', + 'pyarrow' + ] + }, zip_safe=False) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_func_api_classification_binary.py b/tests/test_func_api_classification_binary.py new file mode 100644 index 0000000..a663f5e --- /dev/null +++ b/tests/test_func_api_classification_binary.py @@ -0,0 +1,871 @@ +#------------------------------------------------------------------------------- +# Exactly the same as multiclass but ``n_classes=2`` +# and class name is ``TestFuncClassificationBinary`` +#------------------------------------------------------------------------------- +# !!! cross_val_predict uses stratified split +#------------------------------------------------------------------------------- +# Main concept for testing returned arrays: +# 1). create ground truth e.g. with cross_val_predict +# 2). run vecstack +# 3). compare returned arrays with ground truth +# 4). compare arrays from file with ground truth +#------------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +import unittest +from numpy.testing import assert_array_equal +# from numpy.testing import assert_allclose +from numpy.testing import assert_equal + +import os +import glob +import numpy as np +import scipy.stats as st +from sklearn.model_selection import cross_val_predict +from sklearn.model_selection import cross_val_score +# from sklearn.model_selection import train_test_split +from sklearn.model_selection import StratifiedKFold +from sklearn.datasets import make_classification +from sklearn.metrics import accuracy_score +from sklearn.metrics import log_loss +from sklearn.metrics import make_scorer +from sklearn.linear_model import LogisticRegression +from sklearn.naive_bayes import GaussianNB +from vecstack import stacking + +from sklearn.multiclass import OneVsRestClassifier + +n_classes = 2 +n_folds = 5 +temp_dir = 'tmpdw35lg54ms80eb42' + +X, y = make_classification(n_samples = 500, n_features = 5, n_informative = 3, n_redundant = 1, + n_classes = n_classes, flip_y = 0, random_state = 0) +# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0) + + +# Make train/test split by hand to avoid strange errors probably related to testing suit: +# https://github.com/scikit-learn/scikit-learn/issues/1684 +# https://github.com/scikit-learn/scikit-learn/issues/1704 +# Note: Python 2.7, 3.4 - OK, but 3.5, 3.6 - error + +np.random.seed(0) +ind = np.arange(500) +np.random.shuffle(ind) + +ind_train = ind[:400] +ind_test = ind[400:] + +X_train = X[ind_train] +X_test = X[ind_test] + +y_train = y[ind_train] +y_test = y[ind_test] + + +# Create 4-dim data +np.random.seed(42) +X_train_4d = np.random.normal(size=(400, 8, 8, 3)) +X_test_4d = np.random.normal(size=(100, 8, 8, 3)) +y_train_4d = np.random.randint(n_classes, size=400) + +# Reshape 4-dim to 2-dim +X_train_4d_unrolled = X_train_4d.reshape(X_train_4d.shape[0], -1) +X_test_4d_unrolled = X_test_4d.reshape(X_test_4d.shape[0], -1) + +#------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ + +class LogisticRegressionUnrolled(LogisticRegression): + """ + For tests related to N-dim input. + Estimator accepts N-dim array and reshape it to 2-dim array + """ + def fit(self, X, y): + return super(LogisticRegressionUnrolled, self).fit(X.reshape(X.shape[0], -1), y) + + def predict(self, X): + return super(LogisticRegressionUnrolled, self).predict(X.reshape(X.shape[0], -1)) + + def predict_proba(self, X): + return super(LogisticRegressionUnrolled, self).predict_proba(X.reshape(X.shape[0], -1)) + + +class OneVsRestClassifierUnrolled(OneVsRestClassifier): + """ + Just to avoid data shape checks + """ + def fit(self, X, y): + return super(OneVsRestClassifierUnrolled, self).fit(X.reshape(X.shape[0], -1), y) + + def predict(self, X): + return super(OneVsRestClassifierUnrolled, self).predict(X.reshape(X.shape[0], -1)) + + def predict_proba(self, X): + return super(OneVsRestClassifierUnrolled, self).predict_proba(X.reshape(X.shape[0], -1)) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +class TestFuncClassificationBinary(unittest.TestCase): + + @classmethod + def setUpClass(cls): + try: + os.mkdir(temp_dir) + except: + print('Unable to create temp dir') + + @classmethod + def tearDownClass(cls): + try: + os.rmdir(temp_dir) + except: + print('Unable to remove temp dir') + + def tearDown(self): + # Remove files after each test + files = glob.glob(os.path.join(temp_dir, '*.npy')) + files.extend(glob.glob(os.path.join(temp_dir, '*.log.txt'))) + try: + for file in files: + os.remove(file) + except: + print('Unable to remove temp file') + + #--------------------------------------------------------------------------- + # Test returned and saved arrays in each mode (parameter ) + # Here we also test parameter + #--------------------------------------------------------------------------- + + #--------------------------------------------------------------------------- + # Predict labels + #--------------------------------------------------------------------------- + + def test_oof_pred_mode(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_mode(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + S_test_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_mode(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = None + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_bag_mode(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + S_train_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Predict proba + #--------------------------------------------------------------------------- + + def test_oof_pred_mode_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + _ = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'oof_pred', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_mode_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + S_test_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'oof', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_mode_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = None + _ = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'pred', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode_proba(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True, needs_proba = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + #@@@@ + # Look at proba + # print('\nOne model') + # print('etalon') + # print(S_test_1[:2]) + # print('vecstack') + # print(S_test_2[:2]) + #@@@@ + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_bag_mode_proba(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + S_train_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred_bag', random_state = 0, verbose = 0, stratified = True, needs_proba = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test and parameters + #--------------------------------------------------------------------------- + + def test_oof_pred_bag_mode_shuffle(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 0) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + # !!! Important. Here we pass CV-generator not number of folds + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = kf, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = True, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test parameter and its default values depending on parameter + # Labels + # Important. We use in for any error function + # because we need raw scores (without minus sign) + #--------------------------------------------------------------------------- + def test_oof_mode_metric(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(accuracy_score) + scores = cross_val_score(model, X_train, y = y_train, cv = n_folds, + scoring = scorer, n_jobs = 1, verbose = 0) + mean_str_1 = '%.8f' % np.mean(scores) + std_str_1 = '%.8f' % np.std(scores) + + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0, stratified = True) + + # Load mean score and std from file + # Normally if cleaning is performed there is only one .log.txt file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.log.txt')))[-1] # take the latest file + with open(file_name) as f: + for line in f: + if 'MEAN' in line: + split = line.strip().split() + break + + mean_str_2 = split[1][1:-1] + std_str_2 = split[3][1:-1] + + assert_equal(mean_str_1, mean_str_2) + assert_equal(std_str_1, std_str_2) + + #--------------------------------------------------------------------------- + # Test parameter and its default values depending on parameter + # Proba + # Important. We use in for any error function + # because we need raw scores (without minus sign) + #--------------------------------------------------------------------------- + def test_oof_mode_metric_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(log_loss, response_method='predict_proba') + scores = cross_val_score(model, X_train, y = y_train, cv = n_folds, + scoring = scorer, n_jobs = 1, verbose = 0) + mean_str_1 = '%.8f' % np.mean(scores) + std_str_1 = '%.8f' % np.std(scores) + + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0, stratified = True, + needs_proba = True) + + # Load mean score and std from file + # Normally if cleaning is performed there is only one .log.txt file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.log.txt')))[-1] # take the latest file + with open(file_name) as f: + for line in f: + if 'MEAN' in line: + split = line.strip().split() + break + + mean_str_2 = split[1][1:-1] + std_str_2 = split[3][1:-1] + + assert_equal(mean_str_1, mean_str_2) + assert_equal(std_str_1, std_str_2) + + #------------------------------------------------------------------------------- + # Test several mdels in one run + #------------------------------------------------------------------------------- + + def test_oof_pred_mode_2_models(self): + + # Model a + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1_a = model.predict(X_test).reshape(-1, 1) + + # Model b + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1_b = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode_2_models(self): + + # Model a + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_a = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + # Model b + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = GaussianNB() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_b = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + + def test_oof_pred_mode_proba_2_models(self): + + # Model a + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + _ = model.fit(X_train, y_train) + S_test_1_a = model.predict_proba(X_test) + + # Model b + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + _ = model.fit(X_train, y_train) + S_test_1_b = model.predict_proba(X_test) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'oof_pred', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + + def test_oof_pred_bag_mode_proba_2_models(self): + + # Model a + S_test_1_a = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_a[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + + # Model b + S_test_1_b = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = GaussianNB() + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_b[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True, needs_proba = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + #@@@@ + # Look at proba + # print('\nTwo models') + # print('etalon') + # print(S_test_1[:2]) + # print('vecstack') + # print(S_test_2[:2]) + #@@@@ + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_N_dim_input(self): + """ + This is `test_oof_pred_bag_mode` function with `LogisticRegressionUnrolled` estimator + """ + S_test_temp = np.zeros((X_test_4d_unrolled.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train_4d_unrolled, y_train_4d)): + # Split data and target + X_tr = X_train_4d_unrolled[tr_index] + y_tr = y_train_4d[tr_index] + X_te = X_train_4d_unrolled[te_index] + y_te = y_train_4d[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test_4d_unrolled) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train_4d_unrolled, y = y_train_4d, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [OneVsRestClassifierUnrolled(LogisticRegressionUnrolled(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train_4d, y_train_4d, X_test_4d, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + diff --git a/tests/test_func_api_classification_multiclass.py b/tests/test_func_api_classification_multiclass.py new file mode 100644 index 0000000..db2ae54 --- /dev/null +++ b/tests/test_func_api_classification_multiclass.py @@ -0,0 +1,869 @@ +#------------------------------------------------------------------------------- +# !!! cross_val_predict uses stratified split +#------------------------------------------------------------------------------- +# Main concept for testing returned arrays: +# 1). create ground truth e.g. with cross_val_predict +# 2). run vecstack +# 3). compare returned arrays with ground truth +# 4). compare arrays from file with ground truth +#------------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +import unittest +from numpy.testing import assert_array_equal +# from numpy.testing import assert_allclose +from numpy.testing import assert_equal + +import os +import glob +import numpy as np +import scipy.stats as st +from sklearn.model_selection import cross_val_predict +from sklearn.model_selection import cross_val_score +# from sklearn.model_selection import train_test_split +from sklearn.model_selection import StratifiedKFold +from sklearn.datasets import make_classification +from sklearn.metrics import accuracy_score +from sklearn.metrics import log_loss +from sklearn.metrics import make_scorer +from sklearn.linear_model import LogisticRegression +from sklearn.naive_bayes import GaussianNB +from vecstack import stacking + +from sklearn.multiclass import OneVsRestClassifier + +n_classes = 3 +n_folds = 5 +temp_dir = 'tmpdw35lg54ms80eb42' + +X, y = make_classification(n_samples = 500, n_features = 5, n_informative = 3, n_redundant = 1, + n_classes = n_classes, flip_y = 0, random_state = 0) +# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0) + + +# Make train/test split by hand to avoid strange errors probably related to testing suit: +# https://github.com/scikit-learn/scikit-learn/issues/1684 +# https://github.com/scikit-learn/scikit-learn/issues/1704 +# Note: Python 2.7, 3.4 - OK, but 3.5, 3.6 - error + +np.random.seed(0) +ind = np.arange(500) +np.random.shuffle(ind) + +ind_train = ind[:400] +ind_test = ind[400:] + +X_train = X[ind_train] +X_test = X[ind_test] + +y_train = y[ind_train] +y_test = y[ind_test] + + +# Create 4-dim data +np.random.seed(42) +X_train_4d = np.random.normal(size=(400, 8, 8, 3)) +X_test_4d = np.random.normal(size=(100, 8, 8, 3)) +y_train_4d = np.random.randint(n_classes, size=400) + +# Reshape 4-dim to 2-dim +X_train_4d_unrolled = X_train_4d.reshape(X_train_4d.shape[0], -1) +X_test_4d_unrolled = X_test_4d.reshape(X_test_4d.shape[0], -1) + +#------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ + +class LogisticRegressionUnrolled(LogisticRegression): + """ + For tests related to N-dim input. + Estimator accepts N-dim array and reshape it to 2-dim array + """ + def fit(self, X, y): + return super(LogisticRegressionUnrolled, self).fit(X.reshape(X.shape[0], -1), y) + + def predict(self, X): + return super(LogisticRegressionUnrolled, self).predict(X.reshape(X.shape[0], -1)) + + def predict_proba(self, X): + return super(LogisticRegressionUnrolled, self).predict_proba(X.reshape(X.shape[0], -1)) + + +class OneVsRestClassifierUnrolled(OneVsRestClassifier): + """ + Just to avoid data shape checks + """ + def fit(self, X, y): + return super(OneVsRestClassifierUnrolled, self).fit(X.reshape(X.shape[0], -1), y) + + def predict(self, X): + return super(OneVsRestClassifierUnrolled, self).predict(X.reshape(X.shape[0], -1)) + + def predict_proba(self, X): + return super(OneVsRestClassifierUnrolled, self).predict_proba(X.reshape(X.shape[0], -1)) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +class TestFuncClassificationMulticlass(unittest.TestCase): + + @classmethod + def setUpClass(cls): + try: + os.mkdir(temp_dir) + except: + print('Unable to create temp dir') + + @classmethod + def tearDownClass(cls): + try: + os.rmdir(temp_dir) + except: + print('Unable to remove temp dir') + + def tearDown(self): + # Remove files after each test + files = glob.glob(os.path.join(temp_dir, '*.npy')) + files.extend(glob.glob(os.path.join(temp_dir, '*.log.txt'))) + try: + for file in files: + os.remove(file) + except: + print('Unable to remove temp file') + + #--------------------------------------------------------------------------- + # Test returned and saved arrays in each mode (parameter ) + # Here we also test parameter + #--------------------------------------------------------------------------- + + #--------------------------------------------------------------------------- + # Predict labels + #--------------------------------------------------------------------------- + + def test_oof_pred_mode(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_mode(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + S_test_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_mode(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = None + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_bag_mode(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + S_train_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Predict proba + #--------------------------------------------------------------------------- + + def test_oof_pred_mode_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + _ = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'oof_pred', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_mode_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + S_test_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'oof', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_mode_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = None + _ = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'pred', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode_proba(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True, needs_proba = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + #@@@@ + # Look at proba + # print('\nOne model') + # print('etalon') + # print(S_test_1[:2]) + # print('vecstack') + # print(S_test_2[:2]) + #@@@@ + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_bag_mode_proba(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + S_train_1 = None + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred_bag', random_state = 0, verbose = 0, stratified = True, needs_proba = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test and parameters + #--------------------------------------------------------------------------- + + def test_oof_pred_bag_mode_shuffle(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 0) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + # !!! Important. Here we pass CV-generator not number of folds + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = kf, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = True, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test parameter and its default values depending on parameter + # Labels + # Important. We use in for any error function + # because we need raw scores (without minus sign) + #--------------------------------------------------------------------------- + def test_oof_mode_metric(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(accuracy_score) + scores = cross_val_score(model, X_train, y = y_train, cv = n_folds, + scoring = scorer, n_jobs = 1, verbose = 0) + mean_str_1 = '%.8f' % np.mean(scores) + std_str_1 = '%.8f' % np.std(scores) + + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0, stratified = True) + + # Load mean score and std from file + # Normally if cleaning is performed there is only one .log.txt file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.log.txt')))[-1] # take the latest file + with open(file_name) as f: + for line in f: + if 'MEAN' in line: + split = line.strip().split() + break + + mean_str_2 = split[1][1:-1] + std_str_2 = split[3][1:-1] + + assert_equal(mean_str_1, mean_str_2) + assert_equal(std_str_1, std_str_2) + + #--------------------------------------------------------------------------- + # Test parameter and its default values depending on parameter + # Proba + # Important. We use in for any error function + # because we need raw scores (without minus sign) + #--------------------------------------------------------------------------- + def test_oof_mode_metric_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(log_loss, response_method='predict_proba') + scores = cross_val_score(model, X_train, y = y_train, cv = n_folds, + scoring = scorer, n_jobs = 1, verbose = 0) + mean_str_1 = '%.8f' % np.mean(scores) + std_str_1 = '%.8f' % np.std(scores) + + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))] + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0, stratified = True, + needs_proba = True) + + # Load mean score and std from file + # Normally if cleaning is performed there is only one .log.txt file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.log.txt')))[-1] # take the latest file + with open(file_name) as f: + for line in f: + if 'MEAN' in line: + split = line.strip().split() + break + + mean_str_2 = split[1][1:-1] + std_str_2 = split[3][1:-1] + + assert_equal(mean_str_1, mean_str_2) + assert_equal(std_str_1, std_str_2) + + #------------------------------------------------------------------------------- + # Test several mdels in one run + #------------------------------------------------------------------------------- + + def test_oof_pred_mode_2_models(self): + + # Model a + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1_a = model.predict(X_test).reshape(-1, 1) + + # Model b + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1_b = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode_2_models(self): + + # Model a + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_a = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + # Model b + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = GaussianNB() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_b = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + + def test_oof_pred_mode_proba_2_models(self): + + # Model a + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + _ = model.fit(X_train, y_train) + S_test_1_a = model.predict_proba(X_test) + + # Model b + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + _ = model.fit(X_train, y_train) + S_test_1_b = model.predict_proba(X_test) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, stratified = True, + mode = 'oof_pred', random_state = 0, verbose = 0, needs_proba = True, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + + def test_oof_pred_bag_mode_proba_2_models(self): + + # Model a + S_test_1_a = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_a[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + + # Model b + S_test_1_b = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = GaussianNB() + _ = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_b[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + + model = GaussianNB() + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict_proba') + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + + + models = [OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')), + GaussianNB()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True, needs_proba = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + #@@@@ + # Look at proba + # print('\nTwo models') + # print('etalon') + # print(S_test_1[:2]) + # print('vecstack') + # print(S_test_2[:2]) + #@@@@ + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + + def test_N_dim_input(self): + """ + This is `test_oof_pred_bag_mode` function with `LogisticRegressionUnrolled` estimator + """ + S_test_temp = np.zeros((X_test_4d_unrolled.shape[0], n_folds)) + # Usind StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train_4d_unrolled, y_train_4d)): + # Split data and target + X_tr = X_train_4d_unrolled[tr_index] + y_tr = y_train_4d[tr_index] + X_te = X_train_4d_unrolled[te_index] + y_te = y_train_4d[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test_4d_unrolled) + S_test_1 = st.mode(S_test_temp, axis = 1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train_4d_unrolled, y = y_train_4d, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [OneVsRestClassifierUnrolled(LogisticRegressionUnrolled(random_state=0, solver='liblinear'))] + S_train_2, S_test_2 = stacking(models, X_train_4d, y_train_4d, X_test_4d, + regression = False, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0, stratified = True) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + diff --git a/tests/test_func_api_regression.py b/tests/test_func_api_regression.py new file mode 100644 index 0000000..29835dc --- /dev/null +++ b/tests/test_func_api_regression.py @@ -0,0 +1,1032 @@ +#------------------------------------------------------------------------------- +# Main concept for testing returned arrays: +# 1). create ground truth e.g. with cross_val_predict +# 2). run vecstack +# 3). compare returned arrays with ground truth +# 4). compare arrays from file with ground truth +#------------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +import unittest +from numpy.testing import assert_array_equal +# from numpy.testing import assert_allclose +from numpy.testing import assert_equal +from numpy.testing import assert_raises +from numpy.testing import assert_warns + +import os +import glob +import numpy as np +from scipy.sparse import csr_matrix +from scipy.sparse import csc_matrix +from scipy.sparse import coo_matrix +from sklearn.model_selection import cross_val_predict +from sklearn.model_selection import cross_val_score +# from sklearn.model_selection import train_test_split +from sklearn.model_selection import KFold +# from sklearn.datasets import load_boston +from sklearn.datasets import fetch_openml +from sklearn.metrics import mean_absolute_error +from sklearn.metrics import make_scorer +from sklearn.linear_model import LinearRegression +from sklearn.linear_model import Ridge +from vecstack import stacking +from vecstack.core import model_action + +n_folds = 5 +temp_dir = 'tmpdw35lg54ms80eb42' + +# boston = load_boston() +boston = fetch_openml(name='boston', version=1, as_frame=False, parser='auto') +# X, y = boston.data, boston.target +X, y = boston.data.astype(float), boston.target.astype(float) +# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0) + + +# Make train/test split by hand to avoid strange errors probably related to testing suit: +# https://github.com/scikit-learn/scikit-learn/issues/1684 +# https://github.com/scikit-learn/scikit-learn/issues/1704 +# Note: Python 2.7, 3.4 - OK, but 3.5, 3.6 - error + +np.random.seed(0) +ind = np.arange(500) +np.random.shuffle(ind) + +ind_train = ind[:400] +ind_test = ind[400:] + +X_train = X[ind_train] +X_test = X[ind_test] + +y_train = y[ind_train] +y_test = y[ind_test] + + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +class MinimalEstimator: + """Has no get_params attribute""" + def __init__(self, random_state=0): + self.random_state = random_state + def __repr__(self): + return 'Demo string from __repr__' + def fit(self, X, y): + return self + def predict(self, X): + return np.ones(X.shape[0]) + def predict_proba(self, X): + return np.zeros(X.shape[0]) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +class TestFuncRegression(unittest.TestCase): + + @classmethod + def setUpClass(cls): + try: + os.mkdir(temp_dir) + except: + print('Unable to create temp dir') + + @classmethod + def tearDownClass(cls): + try: + os.rmdir(temp_dir) + except: + print('Unable to remove temp dir') + + def tearDown(self): + # Remove files after each test + files = glob.glob(os.path.join(temp_dir, '*.npy')) + files.extend(glob.glob(os.path.join(temp_dir, '*.log.txt'))) + try: + for file in files: + os.remove(file) + except: + print('Unable to remove temp file') + + #--------------------------------------------------------------------------- + # Testing returned and saved arrays in each mode + #--------------------------------------------------------------------------- + + def test_oof_pred_mode(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_B_mode(self): + """ 'B' is alias for 'oof_pred' """ + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'B', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_mode(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + S_test_1 = None + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_mode(self): + + model = LinearRegression() + S_train_1 = None + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = LinearRegression() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_A_mode(self): + """ 'A' is alias for 'oof_pred_bag' """ + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = LinearRegression() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'A', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_pred_bag_mode(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = LinearRegression() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + S_train_1 = None + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'pred_bag', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing all ones + #--------------------------------------------------------------------------- + def test_oof_pred_mode_sample_weight_one(self): + + sw = np.ones(len(y_train)) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict', + params = {'sample_weight': sw}).reshape(-1, 1) + _ = model.fit(X_train, y_train, sample_weight = sw) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, + sample_weight = sw) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test all random + #--------------------------------------------------------------------------- + def test_oof_pred_mode_sample_weight_random(self): + + np.random.seed(0) + sw = np.random.rand(len(y_train)) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict', + params = {'sample_weight': sw}).reshape(-1, 1) + _ = model.fit(X_train, y_train, sample_weight = sw) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, + sample_weight = sw) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing and parameters + #--------------------------------------------------------------------------- + + def test_oof_pred_mode_transformations(self): + + model = LinearRegression() + S_train_1 = np.expm1(cross_val_predict(model, X_train, y = np.log1p(y_train), cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict')).reshape(-1, 1) + _ = model.fit(X_train, np.log1p(y_train)) + S_test_1 = np.expm1(model.predict(X_test)).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0, + transform_target = np.log1p, transform_pred = np.expm1) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing parameter + #--------------------------------------------------------------------------- + def test_oof_pred_mode_verbose_1(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + models = [LinearRegression()] + S_train_3, S_test_3 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 1) + + models = [LinearRegression()] + S_train_4, S_test_4 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 2) + + models = [LinearRegression()] + S_train_5, S_test_5 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, + mode = 'oof_pred', random_state = 0, verbose = 0) + + models = [LinearRegression()] + S_train_6, S_test_6 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, + mode = 'oof_pred', random_state = 0, verbose = 1) + + models = [LinearRegression()] + S_train_7, S_test_7 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, + mode = 'oof_pred', random_state = 0, verbose = 2) + + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + assert_array_equal(S_train_1, S_train_4) + assert_array_equal(S_test_1, S_test_4) + + assert_array_equal(S_train_1, S_train_5) + assert_array_equal(S_test_1, S_test_5) + + assert_array_equal(S_train_1, S_train_6) + assert_array_equal(S_test_1, S_test_6) + + assert_array_equal(S_train_1, S_train_7) + assert_array_equal(S_test_1, S_test_7) + + #--------------------------------------------------------------------------- + # Test parameter and its default values depending on parameter + # Important. We use in for any error function + # because we need raw scores (without minus sign) + #--------------------------------------------------------------------------- + def test_oof_mode_metric(self): + + model = LinearRegression() + scorer = make_scorer(mean_absolute_error) + scores = cross_val_score(model, X_train, y = y_train, cv = n_folds, + scoring = scorer, n_jobs = 1, verbose = 0) + mean_str_1 = '%.8f' % np.mean(scores) + std_str_1 = '%.8f' % np.std(scores) + + + models = [LinearRegression()] + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0) + + # Load mean score and std from file + # Normally if cleaning is performed there is only one .log.txt file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.log.txt')))[-1] # take the latest file + with open(file_name) as f: + for line in f: + if 'MEAN' in line: + split = line.strip().split() + break + + mean_str_2 = split[1][1:-1] + std_str_2 = split[3][1:-1] + + assert_equal(mean_str_1, mean_str_2) + assert_equal(std_str_1, std_str_2) + + #------------------------------------------------------------------------------- + # Test several mdels in one run + #------------------------------------------------------------------------------- + + def test_oof_pred_mode_2_models(self): + + model = LinearRegression() + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1_a = model.predict(X_test).reshape(-1, 1) + + model = Ridge(random_state = 0) + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(X_train, y_train) + S_test_1_b = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + models = [LinearRegression(), + Ridge(random_state = 0)] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_bag_mode_2_models(self): + + # Model a + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = LinearRegression() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_a = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + model = LinearRegression() + S_train_1_a = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + # Model b + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = Ridge(random_state = 0) + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_b = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + model = Ridge(random_state = 0) + S_train_1_b = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + + models = [LinearRegression(), + Ridge(random_state = 0)] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing sparse types CSR, CSC, COO + #--------------------------------------------------------------------------- + + def test_oof_pred_mode_sparse_csr(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csr_matrix(X_train), y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(csr_matrix(X_train), y_train) + S_test_1 = model.predict(csr_matrix(X_test)).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, csr_matrix(X_train), y_train, csr_matrix(X_test), + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_mode_sparse_csc(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csc_matrix(X_train), y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(csc_matrix(X_train), y_train) + S_test_1 = model.predict(csc_matrix(X_test)).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, csc_matrix(X_train), y_train, csc_matrix(X_test), + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + def test_oof_pred_mode_sparse_coo(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, coo_matrix(X_train), y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(coo_matrix(X_train), y_train) + S_test_1 = model.predict(coo_matrix(X_test)).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, coo_matrix(X_train), y_train, coo_matrix(X_test), + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing X_train -> SCR, X_test -> COO + #--------------------------------------------------------------------------- + + def test_oof_pred_mode_sparse_csr_coo(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csr_matrix(X_train), y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(csr_matrix(X_train), y_train) + S_test_1 = model.predict(coo_matrix(X_test)).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, csr_matrix(X_train), y_train, coo_matrix(X_test), + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing X_train -> SCR, X_test -> Dense + #--------------------------------------------------------------------------- + + def test_oof_pred_mode_sparse_csr_dense(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csr_matrix(X_train), y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + _ = model.fit(csr_matrix(X_train), y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, csr_matrix(X_train), y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing X_test=None + #--------------------------------------------------------------------------- + def test_oof_mode_xtest_is_none(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + S_test_1 = None + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, None, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing parameter exceptions + #--------------------------------------------------------------------------- + def test_exceptions(self): + # Empty model list + assert_raises(ValueError, stacking, [], X_train, y_train, X_test) + # Wrong mode + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train, X_test, mode='abc') + # Path does not exist + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train, X_test, save_dir='./As26bV85') + # n_folds is not int + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train, X_test, n_folds='A') + # n_folds is less than 2 + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train, X_test, n_folds=1) + # Wrong verbose value + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train, X_test, verbose=25) + + # Internal function model_action + assert_raises(ValueError, model_action, LinearRegression(), + X_train, y_train, X_test, sample_weight=None, + action='abc', transform=None) + + # X_test is None when mode != 'oof' + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train, None, mode='oof_pred_bag') + + #--------------------------------------------------------------------------- + # Testing parameter warnings + #--------------------------------------------------------------------------- + def test_warnings(self): + # Parameters specific for classification are ignored if regression=True + assert_warns(UserWarning, stacking, [LinearRegression()], + X_train, y_train, X_test, regression=True, + needs_proba=True) + + assert_warns(UserWarning, stacking, [LinearRegression()], + X_train, y_train, X_test, regression=True, + stratified=True) + + assert_warns(UserWarning, stacking, [LinearRegression()], + X_train, y_train, X_test, regression=True, + needs_proba=True, stratified=True) + + #--------------------------------------------------------------------------- + # Test if model has no 'get_params' + #--------------------------------------------------------------------------- + def test_oof_pred_mode_no_get_params(self): + + S_train_1 = np.ones(X_train.shape[0]).reshape(-1, 1) + S_test_1 = np.ones(X_test.shape[0]).reshape(-1, 1) + + models = [MinimalEstimator()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #-------------------------------------------------------------------------- + # Test inconsistent data shape or type + #-------------------------------------------------------------------------- + def test_inconsistent_data(self): + # nan or inf in y + y_train_nan = y_train.copy() + y_train_nan[0] = np.nan + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train_nan, X_test) + + # y has two or more columns + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, np.c_[y_train, y_train], X_test) + + # X_train and y_train shape nismatch + assert_raises(ValueError, stacking, [LinearRegression()], + X_train, y_train[:10], X_test) + + #--------------------------------------------------------------------------- + # Test small input + #--------------------------------------------------------------------------- + + def test_small_input(self): + """ + This is `test_oof_pred_bag_mode` with small input data + Train: 20 examples + Test: 10 examples + """ + S_test_temp = np.zeros((X_test[:10].shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train[:20], y_train[:20])): + # Split data and target + X_tr = X_train[:20][tr_index] + y_tr = y_train[:20][tr_index] + X_te = X_train[:20][te_index] + y_te = y_train[:20][te_index] + model = LinearRegression() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test[:10]) + S_test_1 = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train[:20], y = y_train[:20], cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train[:20], y_train[:20], X_test[:10], + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof_pred_bag', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Mode 'oof', X_test=None + #--------------------------------------------------------------------------- + + def test_oof_mode_with_none(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + S_test_1 = None + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, None, + regression = True, n_folds = n_folds, shuffle = False, save_dir=temp_dir, + mode = 'oof', random_state = 0, verbose = 0) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # All default values (mode='oof_pred_bag') + #--------------------------------------------------------------------------- + + def test_all_defaults(self): + + # Override global n_folds=5, because default value in stacking function is 4 + n_folds=4 + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits = n_folds, shuffle = False, random_state = None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + model = LinearRegression() + _ = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis = 1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y = y_train, cv = n_folds, + n_jobs = 1, verbose = 0, method = 'predict').reshape(-1, 1) + + models = [LinearRegression()] + S_train_2, S_test_2 = stacking(models, X_train, y_train, X_test, save_dir=temp_dir) + + # Load OOF from file + # Normally if cleaning is performed there is only one .npy file at given moment + # But if we have no cleaning there may be more then one file so we take the latest + file_name = sorted(glob.glob(os.path.join(temp_dir, '*.npy')))[-1] # take the latest file + S = np.load(file_name, allow_pickle=True) + S_train_3 = S[0] + S_test_3 = S[1] + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + diff --git a/tests/test_sklearn_api_classification_binary.py b/tests/test_sklearn_api_classification_binary.py new file mode 100644 index 0000000..5c52b6c --- /dev/null +++ b/tests/test_sklearn_api_classification_binary.py @@ -0,0 +1,878 @@ +# ----------------------------------------------------------------------------- +# Exactly the same as multiclass but ``n_classes=2`` +# and class name is ``TestSklearnClassificationBinary`` +# ----------------------------------------------------------------------------- +# !!! cross_val_predict uses stratified split +# ----------------------------------------------------------------------------- +# Main concept for testing returned arrays: +# 1). create ground truth e.g. with cross_val_predict +# 2). run vecstack +# 3). compare returned arrays with ground truth +# ----------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +import unittest +from numpy.testing import assert_array_equal +# from numpy.testing import assert_allclose +from numpy.testing import assert_equal + +# import os +# import glob +import numpy as np +import scipy.stats as st +from sklearn.model_selection import cross_val_predict +from sklearn.model_selection import cross_val_score +# from sklearn.model_selection import train_test_split +# from sklearn.model_selection import KFold +from sklearn.model_selection import StratifiedKFold +from sklearn.datasets import make_classification +from sklearn.preprocessing import OneHotEncoder +from sklearn.metrics import accuracy_score +from sklearn.metrics import zero_one_loss +from sklearn.metrics import log_loss +from sklearn.metrics import roc_auc_score +from sklearn.metrics import make_scorer +from sklearn.linear_model import LogisticRegression +from sklearn.naive_bayes import GaussianNB +from sklearn.dummy import DummyClassifier +from vecstack import StackingTransformer + +from sklearn.multiclass import OneVsRestClassifier + +n_classes = 2 +n_folds = 5 +# temp_dir = 'tmpdw35lg54ms80eb42' + +X, y = make_classification(n_samples=500, n_features=5, + n_informative=3, n_redundant=1, + n_classes=n_classes, flip_y=0, + random_state=0) + +# X_train, X_test, y_train, y_test = train_test_split(X, y, +# test_size=0.2, +# random_state=0) + + +# Make train/test split by hand to avoid strange errors probably related to testing suit: +# https://github.com/scikit-learn/scikit-learn/issues/1684 +# https://github.com/scikit-learn/scikit-learn/issues/1704 +# Note: Python 2.7, 3.4 - OK, but 3.5, 3.6 - error + +np.random.seed(0) +ind = np.arange(500) +np.random.shuffle(ind) + +ind_train = ind[:400] +ind_test = ind[400:] + +X_train = X[ind_train] +X_test = X[ind_test] + +y_train = y[ind_train] +y_test = y[ind_test] + + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +def roc_auc_score_universal(y_true, y_pred): + """ROC AUC metric for both binary and multiclass classification. + + Parameters + ---------- + y_true - 1d numpy array + True class labels + y_pred - 2d numpy array + Predicted probabilities for each class + """ + ohe = OneHotEncoder(sparse_output=False) + y_true = ohe.fit_transform(y_true.reshape(-1, 1)) + #@@@@ + if len(y_pred.shape) == 1: + y_pred = np.c_[y_pred, y_pred] + y_pred[:, 0] = 1 - y_pred[:, 1] + #@@@@ + auc_score = roc_auc_score(y_true, y_pred) + return auc_score + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +class TestSklearnClassificationBinary(unittest.TestCase): + + # ------------------------------------------------------------------------- + # Test returned arrays in variant B. Labels + # ------------------------------------------------------------------------- + + def test_variant_B_labels(self): + # reference + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant A. Labels + # ------------------------------------------------------------------------- + + def test_variant_A_labels(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis=1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant B. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_B_proba(self): + # reference + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant A. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_A_proba(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis=1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test ``shuffle`` and ``random_state`` parameters in variant A + #--------------------------------------------------------------------------- + def test_variant_A_proba_shuffle_random_state(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=0) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis=1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + # !!! Important. Here we pass CV-generator ``cv=kf`` not number of folds + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=kf, n_jobs=1, verbose=0, + method='predict_proba') + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=True, + variant='A', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default metric and scores. Labels + # ``metric`` parameter and its default values depends on ``regression`` parameter. + # Important. We use ``greater_is_better=True`` in ``make_scorer`` + # for any error function because we need raw scores (without minus sign) + # ------------------------------------------------------------------------- + def test_default_metric_and_scores_labels(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(accuracy_score) + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test custom metric and scores. Labels + # ------------------------------------------------------------------------- + def test_custom_metric_and_scores_labels(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(zero_one_loss) + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, metric=zero_one_loss, + verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test default metric and scores. Probabilities + # ------------------------------------------------------------------------- + def test_default_metric_and_scores_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(log_loss, response_method='predict_proba') + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test custom metric and scores. Probabilities + # ------------------------------------------------------------------------- + def test_custom_metric_and_scores_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(roc_auc_score_universal, response_method='predict_proba') + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + metric=roc_auc_score_universal, verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant B. Labels + # ------------------------------------------------------------------------- + + def test_variant_B_2_estimators_labels(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_e1 = model.predict(X_test).reshape(-1, 1) + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_e2 = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant B. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_B_2_estimators_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1_e1 = model.predict_proba(X_test) + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1_e2 = model.predict_proba(X_test) + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant A. Labels + # ------------------------------------------------------------------------- + + def test_variant_A_2_estimators_labels(self): + + # Estimator 1 + S_test_temp_e1 = np.zeros((X_test.shape[0], n_folds)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + S_test_temp_e1[:, fold_counter] = model.predict(X_test) + S_test_1_e1 = st.mode(S_test_temp_e1, axis=1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # Estimator 2 + S_test_temp_e2 = np.zeros((X_test.shape[0], n_folds)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = GaussianNB() + model = model.fit(X_tr, y_tr) + S_test_temp_e2[:, fold_counter] = model.predict(X_test) + S_test_1_e2 = st.mode(S_test_temp_e2, axis=1, keepdims=True)[0] + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant A. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_A_2_estimators_proba(self): + + # Estimator 1 + S_test_1_e1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp_e1 = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp_e1[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_e1[:, class_id] = np.mean(S_test_temp_e1[:, class_id::n_classes], axis=1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + + # Estimator 2 + S_test_1_e2 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp_e2 = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = GaussianNB() + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp_e2[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_e2[:, class_id] = np.mean(S_test_temp_e2[:, class_id::n_classes], axis=1) + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default (dummy) classifier. Labels + # ------------------------------------------------------------------------- + def test_variant_B_default_classifier_labels(self): + # reference + model = DummyClassifier(strategy='constant', constant=1) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + stack = StackingTransformer(estimators=None, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default (dummy) classifier. Probabilities + # ------------------------------------------------------------------------- + def test_variant_B_default_classifier_proba(self): + # reference + model = DummyClassifier(strategy='constant', constant=1) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + # fit then transform + stack = StackingTransformer(estimators=None, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + needs_proba=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing ``verbose`` parameter + #--------------------------------------------------------------------------- + def test_variant_B_verbose(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # verbose=0 + # fit then transform + estimators = [('lr', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # verbose=1 + # fit then transform + estimators = [('lr', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=1) + stack = stack.fit(X_train, y_train) + S_train_4 = stack.transform(X_train) + S_test_4 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_5 = stack.fit_transform(X_train, y_train) + S_test_5 = stack.transform(X_test) + + # verbose=2 + # fit then transform + estimators = [('lr', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=2) + stack = stack.fit(X_train, y_train) + S_train_6 = stack.transform(X_train) + S_test_6 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_7 = stack.fit_transform(X_train, y_train) + S_test_7 = stack.transform(X_test) + + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + assert_array_equal(S_train_1, S_train_4) + assert_array_equal(S_test_1, S_test_4) + + assert_array_equal(S_train_1, S_train_5) + assert_array_equal(S_test_1, S_test_5) + + assert_array_equal(S_train_1, S_train_6) + assert_array_equal(S_test_1, S_test_6) + + assert_array_equal(S_train_1, S_train_7) + assert_array_equal(S_test_1, S_test_7) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + diff --git a/tests/test_sklearn_api_classification_multiclass.py b/tests/test_sklearn_api_classification_multiclass.py new file mode 100644 index 0000000..767225c --- /dev/null +++ b/tests/test_sklearn_api_classification_multiclass.py @@ -0,0 +1,931 @@ +# ----------------------------------------------------------------------------- +# !!! cross_val_predict uses stratified split +# ----------------------------------------------------------------------------- +# Main concept for testing returned arrays: +# 1). create ground truth e.g. with cross_val_predict +# 2). run vecstack +# 3). compare returned arrays with ground truth +# ----------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +import unittest +from numpy.testing import assert_array_equal +# from numpy.testing import assert_allclose +from numpy.testing import assert_equal +from numpy.testing import assert_raises + +# import os +# import glob +import numpy as np +import scipy.stats as st +from sklearn.model_selection import cross_val_predict +from sklearn.model_selection import cross_val_score +# from sklearn.model_selection import train_test_split +# from sklearn.model_selection import KFold +from sklearn.model_selection import StratifiedKFold +from sklearn.datasets import make_classification +from sklearn.preprocessing import OneHotEncoder +from sklearn.metrics import accuracy_score +from sklearn.metrics import zero_one_loss +from sklearn.metrics import log_loss +from sklearn.metrics import roc_auc_score +from sklearn.metrics import make_scorer +from sklearn.linear_model import LogisticRegression +from sklearn.naive_bayes import GaussianNB +from sklearn.dummy import DummyClassifier +from vecstack import StackingTransformer + +from sklearn.multiclass import OneVsRestClassifier + +from sklearn.ensemble import ExtraTreesClassifier +from sklearn.ensemble import RandomForestClassifier +from sklearn.ensemble import StackingClassifier +from sklearn.pipeline import Pipeline + + +n_classes = 3 +n_folds = 5 +# temp_dir = 'tmpdw35lg54ms80eb42' + +X, y = make_classification(n_samples=500, n_features=5, + n_informative=3, n_redundant=1, + n_classes=n_classes, flip_y=0, + random_state=0) + +# X_train, X_test, y_train, y_test = train_test_split(X, y, +# test_size=0.2, +# random_state=0) + + +# Make train/test split by hand to avoid strange errors probably related to testing suit: +# https://github.com/scikit-learn/scikit-learn/issues/1684 +# https://github.com/scikit-learn/scikit-learn/issues/1704 +# Note: Python 2.7, 3.4 - OK, but 3.5, 3.6 - error + +np.random.seed(0) +ind = np.arange(500) +np.random.shuffle(ind) + +ind_train = ind[:400] +ind_test = ind[400:] + +X_train = X[ind_train] +X_test = X[ind_test] + +y_train = y[ind_train] +y_test = y[ind_test] + + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +def roc_auc_score_universal(y_true, y_pred): + """ROC AUC metric for both binary and multiclass classification. + + Parameters + ---------- + y_true - 1d numpy array + True class labels + y_pred - 2d numpy array + Predicted probabilities for each class + """ + ohe = OneHotEncoder(sparse_output=False) + y_true = ohe.fit_transform(y_true.reshape(-1, 1)) + auc_score = roc_auc_score(y_true, y_pred) + return auc_score + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +class TestSklearnClassificationMulticlass(unittest.TestCase): + + # ------------------------------------------------------------------------- + # Test returned arrays in variant B. Labels + # ------------------------------------------------------------------------- + + def test_variant_B_labels(self): + # reference + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant A. Labels + # ------------------------------------------------------------------------- + + def test_variant_A_labels(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = st.mode(S_test_temp, axis=1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant B. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_B_proba(self): + # reference + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant A. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_A_proba(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis=1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Test ``shuffle`` and ``random_state`` parameters in variant A + #--------------------------------------------------------------------------- + def test_variant_A_proba_shuffle_random_state(self): + + S_test_1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=0) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1[:, class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis=1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + # !!! Important. Here we pass CV-generator ``cv=kf`` not number of folds + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=kf, n_jobs=1, verbose=0, + method='predict_proba') + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=True, + variant='A', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default metric and scores. Labels + # ``metric`` parameter and its default values depends on ``regression`` parameter. + # Important. We use ``greater_is_better=True`` in ``make_scorer`` + # for any error function because we need raw scores (without minus sign) + # ------------------------------------------------------------------------- + def test_default_metric_and_scores_labels(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(accuracy_score) + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test custom metric and scores. Labels + # ------------------------------------------------------------------------- + def test_custom_metric_and_scores_labels(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(zero_one_loss) + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, metric=zero_one_loss, + verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test default metric and scores. Probabilities + # ------------------------------------------------------------------------- + def test_default_metric_and_scores_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(log_loss, response_method='predict_proba') + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test custom metric and scores. Probabilities + # ------------------------------------------------------------------------- + def test_custom_metric_and_scores_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + scorer = make_scorer(roc_auc_score_universal, response_method='predict_proba') + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + metric=roc_auc_score_universal, verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + assert_array_equal(scores_1, scores_2) + assert_array_equal(scores_1, scores_3) + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant B. Labels + # ------------------------------------------------------------------------- + + def test_variant_B_2_estimators_labels(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_e1 = model.predict(X_test).reshape(-1, 1) + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_e2 = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant B. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_B_2_estimators_proba(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1_e1 = model.predict_proba(X_test) + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1_e2 = model.predict_proba(X_test) + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant A. Labels + # ------------------------------------------------------------------------- + + def test_variant_A_2_estimators_labels(self): + + # Estimator 1 + S_test_temp_e1 = np.zeros((X_test.shape[0], n_folds)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + S_test_temp_e1[:, fold_counter] = model.predict(X_test) + S_test_1_e1 = st.mode(S_test_temp_e1, axis=1, keepdims=True)[0] + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # Estimator 2 + S_test_temp_e2 = np.zeros((X_test.shape[0], n_folds)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = GaussianNB() + model = model.fit(X_tr, y_tr) + S_test_temp_e2[:, fold_counter] = model.predict(X_test) + S_test_1_e2 = st.mode(S_test_temp_e2, axis=1, keepdims=True)[0] + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant A. Probabilities + # ------------------------------------------------------------------------- + + def test_variant_A_2_estimators_proba(self): + + # Estimator 1 + S_test_1_e1 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp_e1 = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp_e1[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_e1[:, class_id] = np.mean(S_test_temp_e1[:, class_id::n_classes], axis=1) + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1_e1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + + # Estimator 2 + S_test_1_e2 = np.zeros((X_test.shape[0], n_classes)) + S_test_temp_e2 = np.zeros((X_test.shape[0], n_folds * n_classes)) + # Using StratifiedKFold because by defauld cross_val_predict uses StratifiedKFold + kf = StratifiedKFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = GaussianNB() + model = model.fit(X_tr, y_tr) + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + S_test_temp_e2[:, col_slice_fold] = model.predict_proba(X_test) + for class_id in range(n_classes): + S_test_1_e2[:, class_id] = np.mean(S_test_temp_e2[:, class_id::n_classes], axis=1) + + model = GaussianNB() + S_train_1_e2 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + + S_train_1 = np.c_[S_train_1_e1, S_train_1_e2] + S_test_1 = np.c_[S_test_1_e1, S_test_1_e2] + + # fit then transform + estimators = [('logit', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear'))), + ('bayes', GaussianNB())] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + stratified=True, needs_proba=True, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default (dummy) classifier. Labels + # ------------------------------------------------------------------------- + def test_variant_B_default_classifier_labels(self): + # reference + model = DummyClassifier(strategy='constant', constant=1) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + stack = StackingTransformer(estimators=None, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default (dummy) classifier. Probabilities + # ------------------------------------------------------------------------- + def test_variant_B_default_classifier_proba(self): + # reference + model = DummyClassifier(strategy='constant', constant=1) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict_proba') + model = model.fit(X_train, y_train) + S_test_1 = model.predict_proba(X_test) + + # fit then transform + stack = StackingTransformer(estimators=None, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + needs_proba=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing ``verbose`` parameter + #--------------------------------------------------------------------------- + def test_variant_B_verbose(self): + + model = OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # verbose=0 + # fit then transform + estimators = [('lr', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # verbose=1 + # fit then transform + estimators = [('lr', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=1) + stack = stack.fit(X_train, y_train) + S_train_4 = stack.transform(X_train) + S_test_4 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_5 = stack.fit_transform(X_train, y_train) + S_test_5 = stack.transform(X_test) + + # verbose=2 + # fit then transform + estimators = [('lr', OneVsRestClassifier(LogisticRegression(random_state=0, solver='liblinear')))] + stack = StackingTransformer(estimators, regression=False, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + stratified=True, verbose=2) + stack = stack.fit(X_train, y_train) + S_train_6 = stack.transform(X_train) + S_test_6 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_7 = stack.fit_transform(X_train, y_train) + S_test_7 = stack.transform(X_test) + + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + assert_array_equal(S_train_1, S_train_4) + assert_array_equal(S_test_1, S_test_4) + + assert_array_equal(S_train_1, S_train_5) + assert_array_equal(S_test_1, S_test_5) + + assert_array_equal(S_train_1, S_train_6) + assert_array_equal(S_test_1, S_test_6) + + assert_array_equal(S_train_1, S_train_7) + assert_array_equal(S_test_1, S_test_7) + + + #-------------------------------------------------------------------------- + # Added 20250921 + # Compare with StackingClassifier + #-------------------------------------------------------------------------- + + def test_compare_with_stackinglassifier_from_sklearn(self): + + estimators = [ + ('et', ExtraTreesClassifier(n_estimators=100, random_state=0)), + ('rf', RandomForestClassifier(n_estimators=100, random_state=0))] + final_estimator = LogisticRegression(random_state=0) + + # vecstack.StackingTransformer + stack = StackingTransformer(estimators=estimators, + regression=False, + variant='B', + n_folds=5, + shuffle=False, + stratified=True, + needs_proba=True) + + steps = [('stack', stack), + ('final_estimator', final_estimator)] + pipe = Pipeline(steps) + y_pred_vecstack = pipe.fit(X_train, y_train).predict_proba(X_test) + + # sklearn.ensemble.StackingClassifier + clf = StackingClassifier(estimators=estimators, + final_estimator=final_estimator, + stack_method='predict_proba') + y_pred_sklearn = clf.fit(X_train, y_train).predict_proba(X_test) + + assert_array_equal(y_pred_vecstack, y_pred_sklearn) + + # Compare transformation + + # Transformation for test set is equal + S_test_vecstack = stack.transform(X_test) + S_test_sklearn = clf.transform(X_test) + assert_array_equal(S_test_vecstack, S_test_sklearn) + + # Transformation for train set set is different because StackingClassifier does not use CV procedure + S_train_vecstack = stack.transform(X_train) + S_train_sklearn = clf.transform(X_train) + assert_raises(AssertionError, assert_array_equal, S_train_vecstack, S_train_sklearn) + + # Instead of CV procedure it just uses models trained on the whole train set + et = ExtraTreesClassifier(random_state=0, n_estimators=100) + rf = RandomForestClassifier(random_state=0, n_estimators=100) + y_pred_et = et.fit(X_train, y_train).predict_proba(X_train) + y_pred_rf = rf.fit(X_train, y_train).predict_proba(X_train) + assert_array_equal(S_train_sklearn, np.hstack([y_pred_et, y_pred_rf])) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + diff --git a/tests/test_sklearn_api_regression.py b/tests/test_sklearn_api_regression.py new file mode 100644 index 0000000..67a2a79 --- /dev/null +++ b/tests/test_sklearn_api_regression.py @@ -0,0 +1,2095 @@ +# ----------------------------------------------------------------------------- +# Main concept for testing returned arrays: +# 1). create ground truth e.g. with cross_val_predict +# 2). run vecstack +# 3). compare returned arrays with ground truth +# ----------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +import unittest +from numpy.testing import assert_array_equal +# from numpy.testing import assert_allclose +from numpy.testing import assert_equal +from numpy.testing import assert_raises +from numpy.testing import assert_warns + +import os +import glob +import numpy as np +from scipy.sparse import csr_matrix +from scipy.sparse import csc_matrix +from scipy.sparse import coo_matrix +from sklearn.base import BaseEstimator +from sklearn.base import RegressorMixin +from sklearn.model_selection import cross_val_predict +from sklearn.model_selection import cross_val_score +# from sklearn.model_selection import train_test_split +from sklearn.model_selection import KFold +from sklearn.model_selection import GridSearchCV +from sklearn.model_selection import RandomizedSearchCV +from sklearn.feature_selection import SelectKBest +from sklearn.feature_selection import f_regression +# from sklearn.datasets import load_boston +from sklearn.datasets import fetch_openml +from sklearn.metrics import mean_absolute_error +from sklearn.metrics import mean_squared_error +from sklearn.metrics import make_scorer +from sklearn.metrics import log_loss +from sklearn.dummy import DummyRegressor +from sklearn.dummy import DummyClassifier +from sklearn.linear_model import LinearRegression +from sklearn.linear_model import LogisticRegression +from sklearn.linear_model import Ridge +from sklearn.tree import DecisionTreeRegressor +from sklearn.tree import DecisionTreeClassifier +from sklearn.utils.estimator_checks import check_estimator +from sklearn.utils.multiclass import type_of_target +# from sklearn.externals import joblib +import joblib +from sklearn.pipeline import Pipeline +from sklearn.pipeline import FeatureUnion +from vecstack import StackingTransformer + +from sklearn.ensemble import ExtraTreesRegressor +from sklearn.ensemble import RandomForestRegressor +from sklearn.ensemble import StackingRegressor + +n_folds = 5 +temp_dir = 'tmpdw35lg54ms80eb42' + +# boston = load_boston() +boston = fetch_openml(name='boston', version=1, as_frame=False, parser='auto') +# X, y = boston.data, boston.target +X, y = boston.data.astype(float), boston.target.astype(float) +# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0) + + +# Make train/test split by hand to avoid strange errors probably related to testing suit: +# https://github.com/scikit-learn/scikit-learn/issues/1684 +# https://github.com/scikit-learn/scikit-learn/issues/1704 +# Note: Python 2.7, 3.4 - OK, but 3.5, 3.6 - error + +np.random.seed(0) +ind = np.arange(500) +np.random.shuffle(ind) + +ind_train = ind[:400] +ind_test = ind[400:] + +X_train = X[ind_train] +X_test = X[ind_test] + +y_train = y[ind_train] +y_test = y[ind_test] + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +def log_loss_mod(y_true, y_pred): + """ + When data is very small, cv split may lead to asituation where + `y_true` does not have all labels seen during training. + Original `log_loss` function raises in this case. + + y_true = np.array([0, 1, 1]) + y_pred = np.array([[0.3, 0.3, 0.4], + [0.3, 0.3, 0.4], + [0.3, 0.3, 0.4]]) + log_loss(y_true, y_pred) # ValueError + log_loss_mod(y_true, y_pred) # OK + """ + try: + return log_loss(y_true, y_pred) + except Exception as e: + shape = y_pred.shape + if len(shape) == 2: + try: + return log_loss(y_true, y_pred, labels=range(shape[1])) + except Exception as e: + return 0.0 + else: + return 0.0 + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +class DummyClassifierNumeric(DummyClassifier): + """ + This class is used as a default classification estimator when the `estimators` parameter is None. + The reason for subclassing is that the original `DummyClassifier` allows for string targets, + while `StackingTransformer` does not. Eventually `StackingTransformer` object created with + original `DummyClassifier` does not pass validation with `check_estimator` function. + """ + def fit(self, X, y, sample_weight=None): + type_of_target(y, raise_unknown=True) + return super().fit(X, y, sample_weight=sample_weight) + +# ----------------------------------------------------------------------------- +# Scikit-learn INcompatible estimator +# ----------------------------------------------------------------------------- + +class IncompatibleEstimator: + """ This estimator is not compatible with scikit-learn API. For tests only. + - has no ``get_params`` and ``set_params`` methods + - can not be cloned when ``safe=True``: ``clone(IncompatibleEstimator(), safe=True)`` raises TypeError + - can not be used in Pipeline + """ + def __init__(self, random_state=0): + self.random_state = random_state + def fit(self, X, y): + return self + def predict(self, X): + return np.ones(X.shape[0]) + def predict_proba(self, X): + return np.zeros(X.shape[0]) + +# ----------------------------------------------------------------------------- +# Scikit-learn compatible estimator wich does not support ``sample_weight`` +# ----------------------------------------------------------------------------- + +class EstimatorWithoutSampleWeight(BaseEstimator, RegressorMixin): + """ This estimator does not support ``sample_weight``. For tests only. + """ + def __init__(self, random_state=0): + self.random_state = random_state + def fit(self, X, y): + return self + def predict(self, X): + return np.ones(X.shape[0]) + def predict_proba(self, X): + return np.zeros(X.shape[0]) + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +class TestSklearnRegression(unittest.TestCase): + + @classmethod + def setUpClass(cls): + try: + os.mkdir(temp_dir) + except: + print('Unable to create temp dir') + + @classmethod + def tearDownClass(cls): + try: + files = glob.glob(os.path.join(temp_dir, '*.*')) + for file in files: + os.remove(file) + os.rmdir(temp_dir) + except: + print('Unable to remove temp file or temp dir') + + # ------------------------------------------------------------------------- + # Main scikit-learn compatibility test + # ------------------------------------------------------------------------- + + def test_sklearn_compatibility(self): + # Check with actual `estimators` + # Ignored checks + expected_failed_checks = { + # Training time + 'check_sample_weight_equivalence_on_dense_data': 'CV scheme is used. Changing the number of samples affects the CV split.', + 'check_sample_weight_equivalence_on_sparse_data': 'CV scheme is used. Changing the number of samples affects the CV split.', + # Prediction time + 'check_methods_sample_order_invariance': 'CV scheme is used. Changing the order of samples affects the CV split.', + 'check_methods_subset_invariance': 'CV scheme is used. Changing the number of samples affects the CV split.', + } + + # One estimator + # Regression + estimators = [('lr', LinearRegression())] + check_estimator(StackingTransformer(estimators=estimators, regression=True), expected_failed_checks=expected_failed_checks) + # Classifiaction (class labels) + estimators = [('logit', LogisticRegression())] + check_estimator(StackingTransformer(estimators=estimators, regression=False), expected_failed_checks=expected_failed_checks) + # Classifiaction (proba) + estimators = [('logit', LogisticRegression())] + check_estimator(StackingTransformer(estimators=estimators, regression=False, needs_proba=True, metric=log_loss_mod), expected_failed_checks=expected_failed_checks) + + # Two estimators + # Regression + estimators = [ + ('lr', LinearRegression()), + ('ridge', Ridge()), + ] + check_estimator(StackingTransformer(estimators=estimators, regression=True), expected_failed_checks=expected_failed_checks) + # Classifiaction (class labels) + estimators = [ + ('logit', LogisticRegression()), + ('svc', DecisionTreeClassifier(random_state=0, max_depth=2)), + ] + check_estimator(StackingTransformer(estimators=estimators, regression=False), expected_failed_checks=expected_failed_checks) + # Classifiaction (proba) + estimators = [ + ('logit', LogisticRegression()), + ('svc', DecisionTreeClassifier(random_state=0, max_depth=2)), + ] + check_estimator(StackingTransformer(estimators=estimators, regression=False, needs_proba=True, metric=log_loss_mod), expected_failed_checks=expected_failed_checks) + + # Check with `estimators=None`. + # In this case we don't need `expected_failed_checks`, + # because Dummy estimators are used by default and they predict all constants regardless the split + # Regression + check_estimator(StackingTransformer()) + # Classifiaction + estimators = [('dummyclf', DummyClassifierNumeric(strategy='constant', constant=1))] + check_estimator(StackingTransformer(estimators=estimators, regression=False)) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant B + # ------------------------------------------------------------------------- + + def test_variant_B(self): + # reference + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test returned arrays in variant A + # ------------------------------------------------------------------------- + + def test_variant_A(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = LinearRegression() + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis=1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Testing ``sample_weight`` all ones + # ------------------------------------------------------------------------- + def test_variant_B_sample_weight_one(self): + + sw = np.ones(len(y_train)) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict', + params={'sample_weight': sw}).reshape(-1, 1) + model = model.fit(X_train, y_train, sample_weight=sw) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train, sample_weight=sw) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train, sample_weight=sw) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test ``sample_weight`` all random + # ------------------------------------------------------------------------- + def test_variant_B_sample_weight_random(self): + + np.random.seed(0) + sw = np.random.rand(len(y_train)) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict', + params={'sample_weight': sw}).reshape(-1, 1) + model = model.fit(X_train, y_train, sample_weight=sw) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train, sample_weight=sw) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train, sample_weight=sw) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing ``transform_target`` and ``transform_pred`` parameters + #--------------------------------------------------------------------------- + + def test_variant_B_transformations(self): + + model = LinearRegression() + S_train_1 = np.expm1(cross_val_predict(model, X_train, + y=np.log1p(y_train), + cv=n_folds, n_jobs=1, + verbose=0, + method='predict')).reshape(-1, 1) + model = model.fit(X_train, np.log1p(y_train)) + S_test_1 = np.expm1(model.predict(X_test)).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + transform_target=np.log1p, + transform_pred=np.expm1, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test ``verbose`` parameter. Variant B + # ------------------------------------------------------------------------- + def test_variant_B_verbose(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # verbose=0 + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # verbose=1 + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=1) + stack = stack.fit(X_train, y_train) + S_train_4 = stack.transform(X_train) + S_test_4 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_5 = stack.fit_transform(X_train, y_train) + S_test_5 = stack.transform(X_test) + + # verbose=2 + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=2) + stack = stack.fit(X_train, y_train) + S_train_6 = stack.transform(X_train) + S_test_6 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_7 = stack.fit_transform(X_train, y_train) + S_test_7 = stack.transform(X_test) + + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + assert_array_equal(S_train_1, S_train_4) + assert_array_equal(S_test_1, S_test_4) + + assert_array_equal(S_train_1, S_train_5) + assert_array_equal(S_test_1, S_test_5) + + assert_array_equal(S_train_1, S_train_6) + assert_array_equal(S_test_1, S_test_6) + + assert_array_equal(S_train_1, S_train_7) + assert_array_equal(S_test_1, S_test_7) + + # ------------------------------------------------------------------------- + # Test ``verbose`` parameter. Variant A + # ------------------------------------------------------------------------- + def test_variant_A_verbose(self): + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = LinearRegression() + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis=1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # verbose=0 + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # verbose=1 + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=1) + stack = stack.fit(X_train, y_train) + S_train_4 = stack.transform(X_train) + S_test_4 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_5 = stack.fit_transform(X_train, y_train) + S_test_5 = stack.transform(X_test) + + # verbose=2 + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=2) + stack = stack.fit(X_train, y_train) + S_train_6 = stack.transform(X_train) + S_test_6 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_7 = stack.fit_transform(X_train, y_train) + S_test_7 = stack.transform(X_test) + + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + assert_array_equal(S_train_1, S_train_4) + assert_array_equal(S_test_1, S_test_4) + + assert_array_equal(S_train_1, S_train_5) + assert_array_equal(S_test_1, S_test_5) + + assert_array_equal(S_train_1, S_train_6) + assert_array_equal(S_test_1, S_test_6) + + assert_array_equal(S_train_1, S_train_7) + assert_array_equal(S_test_1, S_test_7) + + # ------------------------------------------------------------------------- + # Test default metric and scores. 1 estimator + # ``metric`` parameter and its default values depends on ``regression`` parameter. + # Important. We use ``greater_is_better=True`` in ``make_scorer`` + # for any error function because we need raw scores (without minus sign) + # ------------------------------------------------------------------------- + def test_default_metric_and_scores_1_estimator(self): + + model = LinearRegression() + scorer = make_scorer(mean_absolute_error) + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + metric=None, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_array_equal(scores_1, scores_2) + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + mean_3 = stack.mean_std_[0][1] + std_3 = stack.mean_std_[0][2] + + assert_array_equal(scores_1, scores_3) + assert_equal(mean_1, mean_3) + assert_equal(std_1, std_3) + + # ------------------------------------------------------------------------- + # Test custom metric and scores. 1 estimator + # ------------------------------------------------------------------------- + def test_custom_metric_and_scores_1_estimator(self): + + model = LinearRegression() + scorer = make_scorer(mean_squared_error) + scores_1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + metric=mean_squared_error, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + scores_2 = stack.scores_[0].copy() + + # mean and std + mean_1 = np.mean(scores_1) + std_1 = np.std(scores_1) + + mean_2 = stack.mean_std_[0][1] + std_2 = stack.mean_std_[0][2] + + assert_array_equal(scores_1, scores_2) + assert_equal(mean_1, mean_2) + assert_equal(std_1, std_2) + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3 = stack.scores_[0].copy() + + mean_3 = stack.mean_std_[0][1] + std_3 = stack.mean_std_[0][2] + + assert_array_equal(scores_1, scores_3) + assert_equal(mean_1, mean_3) + assert_equal(std_1, std_3) + + # ------------------------------------------------------------------------- + # Test default metric and scores. 2 estimator + # ------------------------------------------------------------------------- + def test_default_metric_and_scores_2_estimators(self): + + scorer = make_scorer(mean_absolute_error) + model = LinearRegression() + scores_1_e1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + model = Ridge(random_state=0) + scores_1_e2 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('lr', LinearRegression()), + ('ridge', Ridge(random_state=0))] + stack = StackingTransformer(estimators, regression=True, + metric=None, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + # create copy to avoid array replacement after refit + scores_2_e1 = stack.scores_[0].copy() + scores_2_e2 = stack.scores_[1].copy() + + # mean and std + mean_1_e1 = np.mean(scores_1_e1) + std_1_e1 = np.std(scores_1_e1) + mean_1_e2 = np.mean(scores_1_e2) + std_1_e2 = np.std(scores_1_e2) + + mean_2_e1 = stack.mean_std_[0][1] + std_2_e1 = stack.mean_std_[0][2] + mean_2_e2 = stack.mean_std_[1][1] + std_2_e2 = stack.mean_std_[1][2] + + assert_array_equal(scores_1_e1, scores_2_e1) + assert_array_equal(scores_1_e2, scores_2_e2) + assert_equal(mean_1_e1, mean_2_e1) + assert_equal(mean_1_e2, mean_2_e2) + assert_equal(std_1_e1, std_2_e1) + assert_equal(std_1_e2, std_2_e2) + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3_e1 = stack.scores_[0].copy() + scores_3_e2 = stack.scores_[1].copy() + + mean_3_e1 = stack.mean_std_[0][1] + std_3_e1 = stack.mean_std_[0][2] + mean_3_e2 = stack.mean_std_[1][1] + std_3_e2 = stack.mean_std_[1][2] + + assert_array_equal(scores_1_e1, scores_3_e1) + assert_array_equal(scores_1_e2, scores_3_e2) + assert_equal(mean_1_e1, mean_3_e1) + assert_equal(mean_1_e2, mean_3_e2) + assert_equal(std_1_e1, std_3_e1) + assert_equal(std_1_e2, std_3_e2) + + # ------------------------------------------------------------------------- + # Test custom metric and scores. 2 estimator + # ------------------------------------------------------------------------- + def test_custom_metric_and_scores_2_estimators(self): + + scorer = make_scorer(mean_squared_error) + model = LinearRegression() + scores_1_e1 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + model = Ridge(random_state=0) + scores_1_e2 = cross_val_score(model, X_train, y=y_train, + cv=n_folds, scoring=scorer, + n_jobs=1, verbose=0) + + # fit then transform + estimators = [('lr', LinearRegression()), + ('ridge', Ridge(random_state=0))] + stack = StackingTransformer(estimators, regression=True, + metric=mean_squared_error, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + # create copy to avoid array replacement after refit + scores_2_e1 = stack.scores_[0].copy() + scores_2_e2 = stack.scores_[1].copy() + + # mean and std + mean_1_e1 = np.mean(scores_1_e1) + std_1_e1 = np.std(scores_1_e1) + mean_1_e2 = np.mean(scores_1_e2) + std_1_e2 = np.std(scores_1_e2) + + mean_2_e1 = stack.mean_std_[0][1] + std_2_e1 = stack.mean_std_[0][2] + mean_2_e2 = stack.mean_std_[1][1] + std_2_e2 = stack.mean_std_[1][2] + + assert_array_equal(scores_1_e1, scores_2_e1) + assert_array_equal(scores_1_e2, scores_2_e2) + assert_equal(mean_1_e1, mean_2_e1) + assert_equal(mean_1_e2, mean_2_e2) + assert_equal(std_1_e1, std_2_e1) + assert_equal(std_1_e2, std_2_e2) + + # fit_transform + # also check refitting already fitted transformer + _ = stack.fit_transform(X_train, y_train) + scores_3_e1 = stack.scores_[0].copy() + scores_3_e2 = stack.scores_[1].copy() + + mean_3_e1 = stack.mean_std_[0][1] + std_3_e1 = stack.mean_std_[0][2] + mean_3_e2 = stack.mean_std_[1][1] + std_3_e2 = stack.mean_std_[1][2] + + assert_array_equal(scores_1_e1, scores_3_e1) + assert_array_equal(scores_1_e2, scores_3_e2) + assert_equal(mean_1_e1, mean_3_e1) + assert_equal(mean_1_e2, mean_3_e2) + assert_equal(std_1_e1, std_3_e1) + assert_equal(std_1_e2, std_3_e2) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant B + # ------------------------------------------------------------------------- + def test_variant_B_2_estimators(self): + + model = LinearRegression() + S_train_1_a = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_a = model.predict(X_test).reshape(-1, 1) + + model = Ridge(random_state=0) + S_train_1_b = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_b = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + # fit then transform + estimators = [('lr', LinearRegression()), + ('ridge', Ridge(random_state=0))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test several estimators in one run. Variant A + # ------------------------------------------------------------------------- + + def test_variant_A_2_estimators(self): + + # Model a + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = LinearRegression() + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_a = np.mean(S_test_temp, axis=1).reshape(-1, 1) + + model = LinearRegression() + S_train_1_a = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # Model b + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = Ridge(random_state=0) + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1_b = np.mean(S_test_temp, axis=1).reshape(-1, 1) + + model = Ridge(random_state=0) + S_train_1_b = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_a, S_train_1_b] + S_test_1 = np.c_[S_test_1_a, S_test_1_b] + + # fit then transform + estimators = [('lr', LinearRegression()), + ('ridge', Ridge(random_state=0))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + + # ------------------------------------------------------------------------- + # Testing sparse types CSR, CSC, COO + # ------------------------------------------------------------------------- + + def test_variant_B_sparse_csr(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csr_matrix(X_train), y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(csr_matrix(X_train), y_train) + S_test_1 = model.predict(csr_matrix(X_test)).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(csr_matrix(X_train), y_train) + S_train_2 = stack.transform(csr_matrix(X_train)) + S_test_2 = stack.transform(csr_matrix(X_test)) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(csr_matrix(X_train), y_train) + S_test_3 = stack.transform(csr_matrix(X_test)) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def test_variant_B_sparse_csc(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csc_matrix(X_train), y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(csc_matrix(X_train), y_train) + S_test_1 = model.predict(csc_matrix(X_test)).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(csc_matrix(X_train), y_train) + S_train_2 = stack.transform(csc_matrix(X_train)) + S_test_2 = stack.transform(csc_matrix(X_test)) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(csc_matrix(X_train), y_train) + S_test_3 = stack.transform(csc_matrix(X_test)) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def test_variant_B_sparse_coo(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, coo_matrix(X_train), y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(coo_matrix(X_train), y_train) + S_test_1 = model.predict(coo_matrix(X_test)).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(coo_matrix(X_train), y_train) + S_train_2 = stack.transform(coo_matrix(X_train)) + S_test_2 = stack.transform(coo_matrix(X_test)) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(coo_matrix(X_train), y_train) + S_test_3 = stack.transform(coo_matrix(X_test)) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #--------------------------------------------------------------------------- + # Testing X_train -> SCR, X_test -> COO + #--------------------------------------------------------------------------- + def test_variant_B_sparse_csr_coo(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csr_matrix(X_train), y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(csr_matrix(X_train), y_train) + S_test_1 = model.predict(coo_matrix(X_test)).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(csr_matrix(X_train), y_train) + S_train_2 = stack.transform(csr_matrix(X_train)) + S_test_2 = stack.transform(coo_matrix(X_test)) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(csr_matrix(X_train), y_train) + S_test_3 = stack.transform(coo_matrix(X_test)) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Testing X_train -> SCR, X_test -> Dense + # ------------------------------------------------------------------------- + def test_variant_B_sparse_csr_dense(self): + + model = LinearRegression() + S_train_1 = cross_val_predict(model, csr_matrix(X_train), y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(csr_matrix(X_train), y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(csr_matrix(X_train), y_train) + S_train_2 = stack.transform(csr_matrix(X_train)) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(csr_matrix(X_train), y_train) + S_test_3 = stack.transform(X_test) + + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test default (dummy) regressor + # ------------------------------------------------------------------------- + def test_variant_B_default_regressor(self): + # reference + model = DummyRegressor(strategy='constant', constant=5.5) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + stack = StackingTransformer(estimators=None, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + #------------------------------------------------------------------------------- + # Test inconsistent data shape or type + #------------------------------------------------------------------------------- + def test_inconsistent_data(self): + + stack = StackingTransformer() + + # ---- + + y_train_nan = y_train.copy() + y_train_nan[0] = np.nan + + # nan or inf in y + assert_raises(ValueError, stack.fit, X_train, y_train_nan) + + # y has two or more columns + assert_raises(ValueError, stack.fit, X_train, np.c_[y_train, y_train]) + + # X and y 1st dimention mismatch + assert_raises(ValueError, stack.fit, X_train, y_train[:10]) + + # ---- + + sample_weight = np.random.rand(len(y_train)) + sample_weight_nan = sample_weight.copy() + sample_weight_nan[0] = np.nan + + # nan or inf in sample_weight + assert_raises(ValueError, stack.fit, X_train, y_train, sample_weight_nan) + + # sample_weight has two or more columns + assert_raises(ValueError, stack.fit, X_train, y_train, np.c_[sample_weight, sample_weight]) + + # X and sample_weight 1st dimention mismatch + assert_raises(ValueError, stack.fit, X_train, y_train, sample_weight[:10]) + + # ------------------------------------------------------------------------- + # Inconsistent estimator names + # ------------------------------------------------------------------------- + def test_inconsistent_estimator_names(self): + + # Names are not unique + estimators = [('lr', LinearRegression()), + ('lr', Ridge(random_state=0))] + + stack = StackingTransformer(estimators) + assert_raises(ValueError, stack.fit, X_train, y_train) + + # Names conflict with constructor arguments + estimators = [('lr', LinearRegression()), + ('n_folds', Ridge(random_state=0))] + + stack = StackingTransformer(estimators) + assert_raises(ValueError, stack.fit, X_train, y_train) + + # Names contain double underscore + estimators = [('lr', LinearRegression()), + ('ridge__ridge', Ridge(random_state=0))] + + stack = StackingTransformer(estimators) + assert_raises(ValueError, stack.fit, X_train, y_train) + + # ------------------------------------------------------------------------- + # Testing parameter exceptions + # ------------------------------------------------------------------------- + def test_parameter_exceptions(self): + + # Wrong variant + stack = StackingTransformer(variant='C') + assert_raises(ValueError, stack.fit, X_train, y_train) + + # n_folds is not int + stack = StackingTransformer(n_folds='4') + assert_raises(ValueError, stack.fit, X_train, y_train) + + # n_folds is less than 2 + stack = StackingTransformer(n_folds=1) + assert_raises(ValueError, stack.fit, X_train, y_train) + + # Wrong verbose value + stack = StackingTransformer(verbose=25) + assert_raises(ValueError, stack.fit, X_train, y_train) + + # Internal function _estimator_action + stack = StackingTransformer() + assert_raises(ValueError, stack._estimator_action, + LinearRegression(), X_train, y_train, X_test, + sample_weight=None, action='abc', transform=None) + + # Estimator list is empty + stack = StackingTransformer(estimators=[]) + assert_raises(ValueError, stack.fit, X_train, y_train) + + # Estimator does not support ``sample_weight`` + estimators = [('no_sample_weight', EstimatorWithoutSampleWeight())] + stack = StackingTransformer(estimators) + smaple_weight = np.random.rand(len(y_train)) + # If we do NOT pass sample_weight to StackingTransfrmer.fit it do NOT raise + # (normally works with this estimator) + assert_raises(AssertionError, assert_raises, ValueError, stack.fit, X_train, y_train) + # If we pass sample_weight to StackingTransfrmer.fit it raises + assert_raises(ValueError, stack.fit, X_train, y_train, smaple_weight) + + # ------------------------------------------------------------------------- + # Testing parameter warnings + # ------------------------------------------------------------------------- + def test_parameter_warnings(self): + # Parameters specific for classification are ignored if regression=True + stack = StackingTransformer(regression=True, needs_proba=True) + assert_warns(UserWarning, stack.fit, X_train, y_train) + + stack = StackingTransformer(regression=True, stratified=True) + assert_warns(UserWarning, stack.fit, X_train, y_train) + + stack = StackingTransformer(regression=True, needs_proba=True, stratified=True) + assert_warns(UserWarning, stack.fit, X_train, y_train) + + # ------------------------------------------------------------------------- + # Test incompatible estimator + # ------------------------------------------------------------------------- + def test_incompatible_estimator(self): + estimators = [('ie', IncompatibleEstimator())] + stack = StackingTransformer(estimators) + assert_raises(TypeError, stack.fit, X_train, y_train) + + # ------------------------------------------------------------------------- + # Test ability of fitted StackingTransformer instance to be pickled + # ------------------------------------------------------------------------- + def test_pickle(self): + # reference + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + + # save + _ = joblib.dump(stack, os.path.join(temp_dir, 'stack.pkl')) + + # load + stack_loaded = joblib.load(os.path.join(temp_dir, 'stack.pkl')) + + # transform using loaded instance + S_train_2 = stack_loaded.transform(X_train) + S_test_2 = stack_loaded.transform(X_test) + + # refit loaded instance with fit_transform + S_train_3 = stack_loaded.fit_transform(X_train, y_train) + S_test_3 = stack_loaded.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test ``StackingTransformer.is_train_set`` public method + # ------------------------------------------------------------------------- + def test_is_train_set(self): + # fit + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + + assert_equal(stack.is_train_set(X_train), True) + assert_equal(stack.is_train_set(X_test), False) + + # ------------------------------------------------------------------------- + # Test fallback parameter ``is_train_set`` of ``transform`` method + # ------------------------------------------------------------------------- + def test_fallback_parameter_of_transform_method(self): + # reference + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + # We explicitly tell that this is train set and it actually IS train set + # MUST work as usual + S_train_2 = stack.transform(X_train, is_train_set=True) + S_test_2 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + # We explicitly tell that this is train set but it actually is NOT train set (different shape) + # MUST raise + assert_raises(ValueError, stack.transform, X_train[:10], is_train_set=True) + + # ------------------------------------------------------------------------- + # Test Pipeline and ability to set parameters of nested estimators + # ------------------------------------------------------------------------- + def test_pipeline(self): + # reference + model = LinearRegression(fit_intercept=True) + S_train_1_lr = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_lr = model.predict(X_test).reshape(-1, 1) + + model = DecisionTreeRegressor(random_state=0, max_depth=2) + S_train_1_ridge = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_ridge = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_lr, S_train_1_ridge] + S_test_1 = np.c_[S_test_1_lr, S_test_1_ridge] + + model = Ridge(random_state=0, alpha=2) + model = model.fit(S_train_1, y_train) + y_pred_1 = model.predict(S_test_1) + + # We intentionally set different parameter values to reset them + # later using ``set_params`` method + # We have 4 parameters which differs from reference: + # ``fit_intercept``, ``max_depth``, ``variant``, and ``alpha`` + estimators = [('lr', LinearRegression(fit_intercept=False)), + ('tree', DecisionTreeRegressor(random_state=0, max_depth=4))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + ridge = Ridge(random_state=0, alpha=7) + + steps = [('stack', stack), + ('ridge', ridge)] + + pipe = Pipeline(steps) + + pipe = pipe.fit(X_train, y_train) + y_pred_2 = pipe.predict(X_test) + + # Here we expect that final predictions are different + # because we've set different parameters + assert_raises(AssertionError, assert_array_equal, y_pred_1, y_pred_2) + + # Reset original parameters used in reference + pipe = pipe.set_params(stack__lr__fit_intercept=True, + stack__tree__max_depth=2, + stack__variant='B', + ridge__alpha=2) + + pipe = pipe.fit(X_train, y_train) + y_pred_3 = pipe.predict(X_test) + + # Here we expect that final predictions are equal + assert_array_equal(y_pred_1, y_pred_3) + + + # ------------------------------------------------------------------------- + # Added 20250921 + # Test Pipeline and ability to reset the whole `estimators` collection + # ------------------------------------------------------------------------- + def test_pipeline_2_reset_whole_estimators_collection(self): + # reference + model = LinearRegression(fit_intercept=True) + S_train_1_lr = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_lr = model.predict(X_test).reshape(-1, 1) + + model = DecisionTreeRegressor(random_state=0, max_depth=2) + S_train_1_ridge = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_ridge = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_lr, S_train_1_ridge] + S_test_1 = np.c_[S_test_1_lr, S_test_1_ridge] + + model = Ridge(random_state=0, alpha=2) + model = model.fit(S_train_1, y_train) + y_pred_1 = model.predict(S_test_1) + + # We intentionally set different parameter values to reset them + # later using ``set_params`` method + # We have 5 parameters which differs from reference: + # ``fit_intercept``, ``max_depth``, ``variant``, and ``alpha`` + # and the whole ``estimators`` collection + estimators = [('ridge_1', Ridge(alpha=1.0, fit_intercept=True, random_state=0)), + ('ridge_2', Ridge(alpha=0.1, fit_intercept=False, random_state=1))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + ridge = Ridge(random_state=0, alpha=7) + + steps = [('stack', stack), + ('ridge', ridge)] + + pipe = Pipeline(steps) + + pipe = pipe.fit(X_train, y_train) + y_pred_2 = pipe.predict(X_test) + + # Here we expect that final predictions are different + # because we've set different parameters + assert_raises(AssertionError, assert_array_equal, y_pred_1, y_pred_2) + + # Reset original parameters used in reference + # First we replace the whole `estimators` collection with correct estimators, but incorrect params + # and then reset params of each estimator within the collection + estimators = [('lr', LinearRegression(fit_intercept=False)), + ('tree', DecisionTreeRegressor(random_state=0, max_depth=4))] + # It does not matter where `stack__estimators` will be placed + # i.e. first or last in the list of parameters which we need to reset + pipe = pipe.set_params(stack__estimators=estimators, # replace the whole `estimators` collection + stack__lr__fit_intercept=True, + stack__tree__max_depth=2, + stack__variant='B', + ridge__alpha=2) + + pipe = pipe.fit(X_train, y_train) + y_pred_3 = pipe.predict(X_test) + + # Here we expect that final predictions are equal + assert_array_equal(y_pred_1, y_pred_3) + + + # ------------------------------------------------------------------------- + # Added 20250921 + # Test Pipeline and ability to reset an individual eatimator within the `estimators` collection + # ------------------------------------------------------------------------- + def test_pipeline_3_reset_individual_estimator_within_collection(self): + # reference + model = LinearRegression(fit_intercept=True) + S_train_1_lr = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_lr = model.predict(X_test).reshape(-1, 1) + + model = DecisionTreeRegressor(random_state=0, max_depth=2) + S_train_1_ridge = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_ridge = model.predict(X_test).reshape(-1, 1) + + S_train_1 = np.c_[S_train_1_lr, S_train_1_ridge] + S_test_1 = np.c_[S_test_1_lr, S_test_1_ridge] + + model = Ridge(random_state=0, alpha=2) + model = model.fit(S_train_1, y_train) + y_pred_1 = model.predict(S_test_1) + + # We intentionally set different parameter values to reset them + # later using ``set_params`` method + # We have 5 parameters which differs from reference: + # ``fit_intercept``, ``max_depth``, ``variant``, and ``alpha`` + # and the whole ``estimators`` collection + estimators = [('ridge_1', Ridge(alpha=1.0, fit_intercept=True, random_state=0)), + ('tree', DecisionTreeRegressor(random_state=0, max_depth=4))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + ridge = Ridge(random_state=0, alpha=7) + + steps = [('stack', stack), + ('ridge', ridge)] + + pipe = Pipeline(steps) + + pipe = pipe.fit(X_train, y_train) + y_pred_2 = pipe.predict(X_test) + + # Here we expect that final predictions are different + # because we've set different parameters + assert_raises(AssertionError, assert_array_equal, y_pred_1, y_pred_2) + + # Reset original parameters used in reference + # First we replace individual estimator whithin the collection with correct estimator, but incorrect params + # we retain the name "ridge_1" (instead of "lr") bcause it is arbitrary and does not matter, + # and then reset params of each estimator within the collection + # It does not matter where `stack__estimators` will be placed + # i.e. first or last in the list of parameters which we need to reset + pipe = pipe.set_params(stack__ridge_1=LinearRegression(fit_intercept=False), # replace individual estimator within the collection + stack__ridge_1__fit_intercept=True, + stack__tree__max_depth=2, + stack__variant='B', + ridge__alpha=2) + + pipe = pipe.fit(X_train, y_train) + y_pred_3 = pipe.predict(X_test) + + # Here we expect that final predictions are equal + assert_array_equal(y_pred_1, y_pred_3) + + # ------------------------------------------------------------------------- + # Test FeatureUnion and ability to set parameters of nested estimators + # ------------------------------------------------------------------------- + def test_feature_union(self): + # reference + model = LinearRegression(fit_intercept=True) + S_train_1_lr = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_lr = model.predict(X_test).reshape(-1, 1) + + model = Ridge(random_state=0, alpha=2) + S_train_1_ridge = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_ridge = model.predict(X_test).reshape(-1, 1) + + kbest = SelectKBest(score_func=f_regression, k=5) + kbest = kbest.fit(X_train, y_train) + S_train_1_kbest = kbest.transform(X_train) + S_test_1_kbest = kbest.transform(X_test) + + S_train_1 = np.c_[S_train_1_lr, S_train_1_ridge, S_train_1_kbest] + S_test_1 = np.c_[S_test_1_lr, S_test_1_ridge, S_test_1_kbest] + + + # We intentionally set different parameter values to reset them + # later using ``set_params`` method + # We have 4 parameters which differs from reference: + # ``fit_intercept``, ``alpha``, ``variant``, and ``k`` + estimators = [('lr', LinearRegression(fit_intercept=False)), + ('ridge', Ridge(random_state=0, alpha=7))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + kbest = SelectKBest(score_func=f_regression, k=9) + + transformers = [('stack', stack), + ('kbest', kbest)] + + union = FeatureUnion(transformers) + + union = union.fit(X_train, y_train) + S_train_2 = union.transform(X_train) + S_test_2 = union.transform(X_test) + + # Here we expect that final predictions are different + # because we've set different parameters + assert_raises(AssertionError, assert_array_equal, S_train_1, S_train_2) + assert_raises(AssertionError, assert_array_equal, S_test_1, S_test_2) + + # Reset original parameters used in reference + union = union.set_params(stack__lr__fit_intercept=True, + stack__ridge__alpha=2, + stack__variant='B', + kbest__k=5) + + union = union.fit(X_train, y_train) + S_train_3 = union.transform(X_train) + S_test_3 = union.transform(X_test) + + # Here we expect that final predictions are equal + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test Pipeline with FeatureUnion + # ------------------------------------------------------------------------- + def test_pipeline_with_feature_union(self): + # reference + model = LinearRegression(fit_intercept=True) + S_train_1_lr = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_lr = model.predict(X_test).reshape(-1, 1) + + model = Ridge(random_state=0, alpha=2) + S_train_1_ridge = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_ridge = model.predict(X_test).reshape(-1, 1) + + kbest = SelectKBest(score_func=f_regression, k=5) + kbest = kbest.fit(X_train, y_train) + S_train_1_kbest = kbest.transform(X_train) + S_test_1_kbest = kbest.transform(X_test) + + S_train_1 = np.c_[S_train_1_lr, S_train_1_ridge, S_train_1_kbest] + S_test_1 = np.c_[S_test_1_lr, S_test_1_ridge, S_test_1_kbest] + + model = DecisionTreeRegressor(random_state=0, max_depth=2) + model = model.fit(S_train_1, y_train) + y_pred_1 = model.predict(S_test_1) + + # We intentionally set different parameter values to reset them + # later using ``set_params`` method + # We have 5 parameters which differs from reference: + # ``fit_intercept``, ``alpha``, ``variant``, ``k``, and ``max_depth`` + estimators = [('lr', LinearRegression(fit_intercept=False)), + ('ridge', Ridge(random_state=0, alpha=7))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + + kbest = SelectKBest(score_func=f_regression, k=9) + + transformers = [('stack', stack), + ('kbest', kbest)] + + union = FeatureUnion(transformers) + + tree = DecisionTreeRegressor(random_state=0, max_depth=4) + + steps = [('union', union), + ('tree', tree)] + + pipe = Pipeline(steps) + + pipe = pipe.fit(X_train, y_train) + y_pred_2 = pipe.predict(X_test) + + # Here we expect that final predictions are different + # because we've set different parameters + assert_raises(AssertionError, assert_array_equal, y_pred_1, y_pred_2) + + # Reset original parameters used in reference + pipe = pipe.set_params(union__stack__lr__fit_intercept=True, + union__stack__ridge__alpha=2, + union__stack__variant='B', + union__kbest__k=5, + tree__max_depth=2) + + pipe = pipe.fit(X_train, y_train) + y_pred_3 = pipe.predict(X_test) + + # Here we expect that final predictions are equal + assert_array_equal(y_pred_1, y_pred_3) + + # ------------------------------------------------------------------------- + # Test GridSearchCV and RandomizedGridSearchCV with Pipeline and FeatureUnion + # ------------------------------------------------------------------------- + def test_grid_search_with_pipeline_and_feature_union(self): + # reference + model = LinearRegression(fit_intercept=True) + S_train_1_lr = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_lr = model.predict(X_test).reshape(-1, 1) + + model = Ridge(random_state=0, alpha=2) + S_train_1_ridge = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1_ridge = model.predict(X_test).reshape(-1, 1) + + kbest = SelectKBest(score_func=f_regression, k=5) + kbest = kbest.fit(X_train, y_train) + S_train_1_kbest = kbest.transform(X_train) + S_test_1_kbest = kbest.transform(X_test) + + S_train_1 = np.c_[S_train_1_lr, S_train_1_ridge, S_train_1_kbest] + S_test_1 = np.c_[S_test_1_lr, S_test_1_ridge, S_test_1_kbest] + + model = DecisionTreeRegressor(random_state=0, max_depth=2) + model = model.fit(S_train_1, y_train) + y_pred_1 = model.predict(S_test_1) + + # We intentionally set different parameter values to search best in grid + # We have 5 parameters which differs from reference: + # ``fit_intercept``, ``alpha``, ``variant``, ``k``, and ``max_depth`` + estimators = [('lr', LinearRegression(fit_intercept=False)), + ('ridge', Ridge(random_state=0, alpha=7))] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + + kbest = SelectKBest(score_func=f_regression, k=9) + + transformers = [('stack', stack), + ('kbest', kbest)] + + union = FeatureUnion(transformers) + + tree = DecisionTreeRegressor(random_state=0, max_depth=4) + + steps = [('union', union), + ('tree', tree)] + + pipe = Pipeline(steps) + + # Parameter search space + params = {'union__stack__lr__fit_intercept': [True], + 'union__stack__ridge__alpha': [2], + 'union__stack__variant' : ['B'], + 'union__kbest__k': [1, 5], + 'tree__max_depth': [2]} + + # Exhaustive search + grid = GridSearchCV(pipe, params, n_jobs=1) + grid = grid.fit(X_train, y_train) + y_pred_2 = grid.predict(X_test) + + assert_array_equal(y_pred_1, y_pred_2) + + # Random search + # We set such ``n_iter`` that it gives exhaustive search to get the same result as reference + rand_grid = RandomizedSearchCV(pipe, params, random_state=0, n_iter=2) + rand_grid = rand_grid.fit(X_train, y_train) + y_pred_3 = rand_grid.predict(X_test) + + assert_array_equal(y_pred_1, y_pred_3) + + # ------------------------------------------------------------------------- + # Test ``shuffle`` and ``random_state``. Variant B + # ------------------------------------------------------------------------- + def test_variant_B_shuffle_and_random_state(self): + # reference + model = LinearRegression() + kf = KFold(n_splits=n_folds, shuffle=True, random_state=0) + # !!! below we pass KFold object (not a number of folds) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=kf, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=True, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test ``shuffle`` and ``random_state``. Variant A + # ------------------------------------------------------------------------- + def test_variant_A_shuffle_and_random_state(self): + + S_test_temp = np.zeros((X_test.shape[0], n_folds)) + kf = KFold(n_splits=n_folds, shuffle=True, random_state=0) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + # X_te = X_train[te_index] + # y_te = y_train[te_index] + model = LinearRegression() + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test) + S_test_1 = np.mean(S_test_temp, axis=1).reshape(-1, 1) + + model = LinearRegression() + # !!! below we pass KFold object (not a number of folds) + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=kf, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=True, + variant='A', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test ``_get_footprint`` method + # ------------------------------------------------------------------------- + def test_get_footprint(self): + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + # ``X`` argument is correct - must NOT raise + assert_raises(AssertionError, assert_raises, ValueError, stack._get_footprint, X_train) + # ``X`` argument is INcorrect - MUST raise + assert_raises(ValueError, stack._get_footprint, 5) + + # ------------------------------------------------------------------------- + # Test ``_check_identity`` method + # ------------------------------------------------------------------------- + def test_check_identity(self): + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + # ``X`` argument is correct - must NOT raise + assert_raises(AssertionError, assert_raises, ValueError, stack._check_identity, X_train) + # ``X`` argument is INcorrect - MUST raise + assert_raises(ValueError, stack._check_identity, 5) + + # ------------------------------------------------------------------------- + # Test ``_random_choice`` method + # ------------------------------------------------------------------------- + def test_random_choice(self): + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + # very large range - must NOT raise + assert_raises(AssertionError, assert_raises, ValueError, stack._random_choice, 19999999999, 1000) + # ``size`` is less than ``n`` - must NOT raise + assert_raises(AssertionError, assert_raises, ValueError, stack._random_choice, 200, 20) + # ``size`` is greater than ``n`` - MUST raise + assert_raises(ValueError, stack._random_choice, 20, 200) + + # ------------------------------------------------------------------------- + # Test case where X_test has the same shape as X_train + # but contains different data + # ------------------------------------------------------------------------- + def test_x_test_has_same_shape(self): + # reference + X_test_same_shape = np.r_[X_test[:101], X_test[:101], X_test[:101], X_test[:101]] + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train, y=y_train, + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + model = model.fit(X_train, y_train) + S_test_1 = model.predict(X_test_same_shape).reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='B', random_state=0, + verbose=0) + stack = stack.fit(X_train, y_train) + S_train_2 = stack.transform(X_train) + S_test_2 = stack.transform(X_test_same_shape) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train, y_train) + S_test_3 = stack.transform(X_test_same_shape) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Test small input + # ------------------------------------------------------------------------- + + def test_small_input(self): + """ + This is `test_variant_A` with small input data + Train: 20 examples + Test: 10 examples + """ + S_test_temp = np.zeros((X_test[:10].shape[0], n_folds)) + kf = KFold(n_splits=n_folds, shuffle=False, random_state=None) + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train[:20], y_train[:20])): + # Split data and target + X_tr = X_train[:20][tr_index] + y_tr = y_train[:20][tr_index] + # X_te = X_train[:20][te_index] + # y_te = y_train[:20][te_index] + model = LinearRegression() + model = model.fit(X_tr, y_tr) + S_test_temp[:, fold_counter] = model.predict(X_test[:10]) + S_test_1 = np.mean(S_test_temp, axis=1).reshape(-1, 1) + + model = LinearRegression() + S_train_1 = cross_val_predict(model, X_train[:20], y=y_train[:20], + cv=n_folds, n_jobs=1, verbose=0, + method='predict').reshape(-1, 1) + + # fit then transform + estimators = [('lr', LinearRegression())] + stack = StackingTransformer(estimators, regression=True, + n_folds=n_folds, shuffle=False, + variant='A', random_state=0, + verbose=0) + stack = stack.fit(X_train[:20], y_train[:20]) + S_train_2 = stack.transform(X_train[:20]) + S_test_2 = stack.transform(X_test[:10]) + + # fit_transform + # also check refitting already fitted transformer + S_train_3 = stack.fit_transform(X_train[:20], y_train[:20]) + S_test_3 = stack.transform(X_test[:10]) + + # compare + assert_array_equal(S_train_1, S_train_2) + assert_array_equal(S_test_1, S_test_2) + + assert_array_equal(S_train_1, S_train_3) + assert_array_equal(S_test_1, S_test_3) + + # ------------------------------------------------------------------------- + # Added 20250921 + # Compare with StackingRegressor + # ------------------------------------------------------------------------- + def test_compare_with_stackingregressor_from_sklearn(self): + + estimators = [ + ('et', ExtraTreesRegressor(n_estimators=100, random_state=0)), + ('rf', RandomForestRegressor(n_estimators=100, random_state=0))] + final_estimator = LinearRegression() + + # vecstack.StackingTransformer + stack = StackingTransformer(estimators=estimators, + regression=True, + variant='B', + n_folds=5, + shuffle=False) + + steps = [('stack', stack), + ('final_estimator', final_estimator)] + pipe = Pipeline(steps) + y_pred_vecstack = pipe.fit(X_train, y_train).predict(X_test) + + # sklearn.ensemble.StackingClassifier + clf = StackingRegressor(estimators=estimators, + final_estimator=final_estimator) + y_pred_sklearn = clf.fit(X_train, y_train).predict(X_test) + + assert_array_equal(y_pred_vecstack, y_pred_sklearn) + + # Compare transformation + + # Transformation for test set is equal + S_test_vecstack = stack.transform(X_test) + S_test_sklearn = clf.transform(X_test) + assert_array_equal(S_test_vecstack, S_test_sklearn) + + # Transformation for train set set is different because StackingClassifier does not use CV procedure + S_train_vecstack = stack.transform(X_train) + S_train_sklearn = clf.transform(X_train) + assert_raises(AssertionError, assert_array_equal, S_train_vecstack, S_train_sklearn) + + # Instead of CV procedure it just uses models trained on the whole train set + et = ExtraTreesRegressor(random_state=0, n_estimators=100) + rf = RandomForestRegressor(random_state=0, n_estimators=100) + y_pred_et = et.fit(X_train, y_train).predict(X_train) + y_pred_rf = rf.fit(X_train, y_train).predict(X_train) + assert_array_equal(S_train_sklearn, np.hstack([y_pred_et.reshape(-1, 1), y_pred_rf.reshape(-1, 1)])) + + # ------------------------------------------------------------------------- + # Added 20250924 + # Explicitly check that `validate_data` checks number of features + # ------------------------------------------------------------------------- + + def test_inconsistent_shape_passed_to_transform(self): + """ + When transforming non-training set there was a check: + ``` + if X.shape[1] != self.n_features_: + raise ValueError('Inconsistent number of features.') + ``` + It was needed because I used `check_array` function to validate data + and probably number of features was not checked. + + Now I check data with `validate_data` which checks `self.n_features_in_`. + So my manual check can never happen and coverage dropped. + So I removed my manual check and created this test case to confirm explicitly that `validate_data` works. + + In version 0.4.0 there was no specific test for this case, + probably because it was included in `check_estimator`. + """ + estimators = [ + ('lr', LinearRegression()), + ('ridge', Ridge())] + + stack = StackingTransformer(estimators=estimators, + regression=True, + variant='B', + n_folds=5, + shuffle=False) + + stack = stack.fit(X_train, y_train) + S_train = stack.transform(X_train) # OK + S_test = stack.transform(X_test) # OK + + # Transform train set with different number of features - in fact it is identified as non-train set because shape is different + assert_raises(ValueError, stack.transform, X_train[:, 1:]) + + # Transform test set with different number of features + assert_raises(ValueError, stack.transform, X_test[:, :-1]) + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +if __name__ == '__main__': + unittest.main() + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + diff --git a/vecstack/__init__.py b/vecstack/__init__.py index 65e24cb..070be10 100644 --- a/vecstack/__init__.py +++ b/vecstack/__init__.py @@ -1,12 +1,14 @@ """Python package for stacking (machine learning technique) Find out how to use: ->>>from vecstack import stacking ->>>help(stacking) +>>> from vecstack import stacking +>>> help(stacking) +>>> from vecstack import StackingTransformer +>>> help(StackingTransformer) MIT License -Copyright (c) 2016 vecxoz +Copyright (c) 2016-2025 Igor Ivanov Email: vecxoz@gmail.com Permission is hereby granted, free of charge, to any person obtaining a copy @@ -28,7 +30,17 @@ SOFTWARE. """ -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- from .core import stacking +from .coresk import StackingTransformer + +__author__ = 'Igor Ivanov > kaggle.com/vecxoz' +__license__ = 'MIT' +__version__ = '0.5.2' + +__all__ = ['stacking', 'StackingTransformer'] + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- diff --git a/vecstack/core.py b/vecstack/core.py index 4ba52ba..66936d3 100644 --- a/vecstack/core.py +++ b/vecstack/core.py @@ -1,12 +1,12 @@ -"""Python package for stacking (machine learning technique) +"""Functional API for stacking. Find out how to use: ->>>from vecstack import stacking ->>>help(stacking) +>>> from vecstack import stacking +>>> help(stacking) MIT License -Copyright (c) 2016 vecxoz +Copyright (c) 2016-2025 Igor Ivanov Email: vecxoz@gmail.com Permission is hereby granted, free of charge, to any person obtaining a copy @@ -31,12 +31,26 @@ #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- +from __future__ import print_function +from __future__ import division + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +import os +import sys +import warnings +from datetime import datetime import numpy as np import scipy.stats as st from sklearn.model_selection import KFold from sklearn.model_selection import StratifiedKFold from sklearn.metrics import mean_absolute_error from sklearn.metrics import accuracy_score +from sklearn.metrics import log_loss +from sklearn.utils.validation import check_X_y +from sklearn.utils.validation import check_array +from sklearn.base import clone #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- @@ -47,12 +61,73 @@ def transformer(y, func=None): return y else: return func(y) + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +def model_action(model, X_train, y_train, X_test, + sample_weight=None, action=None, + transform=None): + """Performs model action. + This wrapper gives us ability to choose action dynamically + (e.g. predict or predict_proba). + Note. Through and then through we + apply and functions if given by user + on the target and prediction in each fold separately + to be able to calculate proper scores + """ + if 'fit' == action: + # We use following condition, because some models (e.g. Lars) may not have + # 'sample_weight' parameter of fit method + if sample_weight is not None: + return model.fit(X_train, transformer(y_train, func = transform), sample_weight=sample_weight) + else: + return model.fit(X_train, transformer(y_train, func = transform)) + elif 'predict' == action: + return transformer(model.predict(X_test), func = transform) + elif 'predict_proba' == action: + return transformer(model.predict_proba(X_test), func = transform) + else: + raise ValueError('Parameter action must be set properly') #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- -def stacking(models, X_train, y_train, X_test, regression=True, +def model_params(model): + """ + Create string of alphabetically sorted parameters of the model + obtained with get_params or string contaning result of __repr__ call + """ + s = '' + + if hasattr(model, 'get_params'): + params_dict = model.get_params() + max_len = 0 + for key in params_dict: + if len(key) > max_len: + max_len = len(key) + sorted_keys = sorted(params_dict.keys()) + for key in sorted_keys: + s += '%-*s %s\n' % (max_len, key, params_dict[key]) + + elif hasattr(model, '__repr__'): + s = model.__repr__() + s += '\n' + + else: + s = 'Model has no ability to show parameters (has no or <__repr__>)\n' + + s += '\n' + + return s + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- + +def stacking(models, X_train, y_train, X_test, + sample_weight=None, regression=True, transform_target=None, transform_pred=None, + mode='oof_pred_bag', needs_proba=False, save_dir=None, metric=None, n_folds=4, stratified=False, shuffle=False, random_state=0, verbose=0): """Function 'stacking' takes train data, test data and list of 1-st level @@ -64,16 +139,26 @@ def stacking(models, X_train, y_train, X_test, regression=True, ---------- models : list List of 1-st level models. You can use any models that follow sklearn - convention i.e. accept numpy arrays and have methods 'fit' and 'predict'. + convention i.e. accept numpy arrays and have + methods 'fit', 'predict' and 'predict_proba'. + Following sklearn convention in binary classification + task with probabilities model must return probabilities + for each class (i.e. two columns). - X_train : numpy array or sparse matrix of shape [n_train_samples, n_features] + X_train : numpy array or sparse matrix of N-dim shape, e.g. 2-dim [n_train_samples, n_features] Training data y_train : numpy 1d array Target values - X_test : numpy array or sparse matrix of shape [n_test_samples, n_features] + X_test : numpy array or sparse matrix of N-dim shape, e.g. 2-dim [n_test_samples, n_features], or None Test data + Note: X_test can be set to None when mode='oof' + + sample_weight : numpy array of shape [n_train_samples], default None + Individual weights for each sample (passed to fit method of the model). + Note: sample_weight must have the same length as full training set X_train. + It will be split automatically for each fold. regression : boolean, default True If True - perform stacking for regression task, @@ -83,11 +168,11 @@ def stacking(models, X_train, y_train, X_test, regression=True, Function to transform target variable. If None - transformation is not used. For example, for regression task (if target variable is skewed) - you can use transformation like numpy.log. - Set transform_target = numpy.log + you can use transformation like numpy.log1p. + Set transform_target = numpy.log1p Usually you want to use respective backward transformation - for prediction like numpy.exp. - Set transform_pred = numpy.exp + for prediction like numpy.expm1. + Set transform_pred = numpy.expm1 Caution! Some transformations may give inapplicable results. For example, if target variable contains zeros, numpy.log gives you -inf. In such case you can use appropriate @@ -98,52 +183,91 @@ def stacking(models, X_train, y_train, X_test, regression=True, Function to transform prediction. If None - transformation is not used. If you use transformation for target variable (transform_target) - like numpy.log, then using transform_pred you can specify - respective backward transformation like numpy.exp. + like numpy.log1p, then using transform_pred you can specify + respective backward transformation like numpy.expm1. Look at description of parameter transform_target + mode: str, default 'oof_pred_bag' (alias 'A') + Note: for detailes see terminology below + 'oof' - return only oof. X_test can be set to None + 'oof_pred' (alias 'B') - return oof and pred + 'oof_pred_bag' (alias 'A') - return oof and bagged pred + 'pred' - return pred only + 'pred_bag' - return bagged pred only + Terminology: + oof - out-of-fold predictions for train set + pred - predictions for tests set (model is fitted once on full train set, then predicts test set) + bagged pred - bagged predictions for tests set (given that we have N folds, + we fit N models on each fold's train data, then each model predicts test set, + then we perform bagging: compute mean of predicted values (for regression or class probabilities) + or majority voting: compute mode (when predictions are class labels) + + needs_proba: boolean, default False, meaningful only for classification task + Whether to predict probabilities (instead of class labels) + in classification task. + Ignored if regression=True + + save_dir: str, default None + If specified - considered as a valid directory (must exist) where log and + returned arrays will be saved. + If not specified - log and arrays will not be saved. + Path may be absolute or relative to the directory from where script was run. + Absolute examples: Win: 'c:/some/dir', Linux: '/home/user/run' + Relative examples: Win and Linux current directory: '.' + Both arrays are saved in a single .npy file, so you can load it as follows: + S = np.load('c:/some/dir/[2017.11.29].[13.47.31].250824.45dc2b.npy') + S_train = S[0] + S_test = S[1] + Log is saved in plain text. + File names are the current timestamp plus random part to ensure uniqueness. + metric : callable, default None Evaluation metric (score function) which is used to calculate results of cross-validation. If None, then by default: sklearn.metrics.mean_absolute_error - for regression - sklearn.metrics.accuracy_score - for classification + sklearn.metrics.accuracy_score - for classification with class labels + sklearn.metrics.log_loss - for classification with probabilities You can use any appropriate sklearn metric or define your own metric like shown below: def your_metric(y_true, y_pred): # calculate return result + + MEAN/FULL interpretation: + MEAN - mean (average) of scores for each fold. + FULL - metric calculated using combined oof predictions + for full train set and target. + For some metrics (e.g. rmse, rmsle) MEAN and FULL values are + slightly different n_folds : int, default 4 Number of folds in cross-validation stratified : boolean, default False, meaningful only for classification task If True - use stratified folds in cross-validation + Ignored if regression=True shuffle : boolean, default False Whether to perform a shuffle before cross-validation split random_state : int, default 0 - Random seed for shuffle + Random seed + Ignored if shuffle=False verbose : int, default 0 Level of verbosity. 0 - show no messages - 1 - for each model show single mean score + 1 - for each model show mean score 2 - for each model show score for each fold and mean score - Caution. To calculate MEAN score across all folds - full train set prediction and full true target are used. - So for some metrics (e.g. rmse) this value may not be equal - to mean of score values calculated for each fold. - Returns ------- - S_train : numpy array of shape [n_train_samples, n_models] + S_train : numpy array of shape [n_train_samples, n_models] or None Stacking features for train set - S_test : numpy array of shape [n_test_samples, n_models] + S_test : numpy array of shape [n_test_samples, n_models] or None Stacking features for test set Brief example (complete examples - see below) @@ -156,7 +280,7 @@ def your_metric(y_true, y_pred): # Get your stacking features in a single line S_train, S_test = stacking(models, X_train, y_train, X_test, - regression = True, verbose = 2) + regression = True, verbose = 2) # Use 2-nd level model with stacking features @@ -179,7 +303,7 @@ def your_metric(y_true, y_pred): Regression ---------- - from sklearn.datasets import load_boston + from sklearn.datasets import fetch_california_housing from sklearn.model_selection import train_test_split from sklearn.metrics import mean_absolute_error from sklearn.ensemble import ExtraTreesRegressor @@ -188,35 +312,36 @@ def your_metric(y_true, y_pred): from vecstack import stacking # Load demo data - boston = load_boston() - X, y = boston.data, boston.target + X, y = fetch_california_housing(return_X_y=True) # Make train/test split # As usual in machine learning task we have X_train, y_train, and X_test - X_train, X_test, y_train, y_test = train_test_split(X, y, - test_size = 0.2, random_state = 0) + X_train, X_test, y_train, y_test = train_test_split(X, y, + test_size=0.2, + random_state=0) - # Caution! All models and parameter values are just + # Caution! All models and parameter values are just # demonstrational and shouldn't be considered as recommended. # Initialize 1-st level models. models = [ - ExtraTreesRegressor(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), + ExtraTreesRegressor(random_state=0, n_jobs=-1, + n_estimators=100, max_depth=3), - RandomForestRegressor(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), + RandomForestRegressor(random_state=0, n_jobs=-1, + n_estimators=100, max_depth=3), - XGBRegressor(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3)] + XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, + n_estimators=100, max_depth=3)] # Compute stacking features - S_train, S_test = stacking(models, X_train, y_train, X_test, - regression = True, metric = mean_absolute_error, n_folds = 4, - shuffle = True, random_state = 0, verbose = 2) + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression=True, metric=mean_absolute_error, + n_folds=4, shuffle=True, + random_state=0, verbose=2) # Initialize 2-nd level model - model = XGBRegressor(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3) + model = XGBRegressor(random_state=0, n_jobs=-1, learning_rate=0.1, + n_estimators=100, max_depth=3) # Fit 2-nd level model model = model.fit(S_train, y_train) @@ -239,35 +364,36 @@ def your_metric(y_true, y_pred): from vecstack import stacking # Load demo data - iris = load_iris() - X, y = iris.data, iris.target + X, y = load_iris(return_X_y=True) # Make train/test split # As usual in machine learning task we have X_train, y_train, and X_test - X_train, X_test, y_train, y_test = train_test_split(X, y, - test_size = 0.2, random_state = 0) + X_train, X_test, y_train, y_test = train_test_split(X, y, + test_size=0.2, + random_state=0) - # Caution! All models and parameter values are just + # Caution! All models and parameter values are just # demonstrational and shouldn't be considered as recommended. # Initialize 1-st level models. models = [ - ExtraTreesClassifier(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), + ExtraTreesClassifier(random_state=0, n_jobs=-1, + n_estimators=100, max_depth=3), - RandomForestClassifier(random_state = 0, n_jobs = -1, - n_estimators = 100, max_depth = 3), + RandomForestClassifier(random_state=0, n_jobs=-1, + n_estimators=100, max_depth=3), - XGBClassifier(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3)] + XGBClassifier(seed=0, n_jobs=-1, learning_rate=0.1, + n_estimators=100, max_depth=3)] # Compute stacking features - S_train, S_test = stacking(models, X_train, y_train, X_test, - regression = False, metric = accuracy_score, n_folds = 4, - stratified = True, shuffle = True, random_state = 0, verbose = 2) + S_train, S_test = stacking(models, X_train, y_train, X_test, + regression=False, metric=accuracy_score, + n_folds=4, stratified=True, shuffle=True, + random_state=0, verbose=2) # Initialize 2-nd level model - model = XGBClassifier(seed = 0, n_jobs = -1, learning_rate = 0.1, - n_estimators = 100, max_depth = 3) + model = XGBClassifier(seed=0, n_jobs=-1, learning_rate=0.1, + n_estimators=100, max_depth=3) # Fit 2-nd level model model = model.fit(S_train, y_train) @@ -278,67 +404,311 @@ def your_metric(y_true, y_pred): # Final prediction score print('Final prediction score: [%.8f]' % accuracy_score(y_test, y_pred)) """ - # Print type of task - if regression and verbose > 0: - print('task: [regression]') - elif not regression and verbose > 0: - print('task: [classification]') - - # Specify default metric for cross-validation + #--------------------------------------------------------------------------- + # Check parameters + #--------------------------------------------------------------------------- + # If empty list + if 0 == len(models): + raise ValueError('List of models is empty') + # X_test can be None only if mode='oof' + if X_test is None and mode != 'oof': + raise ValueError("X_test can be None only if mode='oof'") + # Check arrays + # y_train and sample_weight must be 1d ndarrays (i.e. row, not column) + X_train, y_train = check_X_y(X_train, + y_train, + accept_sparse=['csr'], # allow csr and cast all other sparse types to csr + ensure_all_finite=False, # allow nan and inf because + # some models (xgboost) can handle + allow_nd=True, + multi_output=False) # do not allow several columns in y_train + + if X_test is not None: # allow X_test to be None for mode='oof' + X_test = check_array(X_test, + accept_sparse=['csr'], # allow csr and cast all other sparse types to csr + allow_nd=True, + ensure_all_finite=False) # allow nan and inf because + # some models (xgboost) can handle + if sample_weight is not None: + sample_weight = np.array(sample_weight).ravel() + # + regression = bool(regression) + # If wrong + if mode not in ['pred', 'pred_bag', 'oof', 'oof_pred', 'B', 'oof_pred_bag', 'A']: + raise ValueError('Parameter must be set properly') + # + needs_proba = bool(needs_proba) + # If wrong + if save_dir is not None: + save_dir = os.path.normpath(save_dir) + if not os.path.isdir(save_dir): + raise ValueError('Path does not exist or is not a directory. Check parameter') + # + if not isinstance(n_folds, int): + raise ValueError('Parameter must be integer') + if not n_folds > 1: + raise ValueError('Parameter must be not less than 2') + # + stratified = bool(stratified) + # + shuffle = bool(shuffle) + # + # To comply with sklearn requirement + if not shuffle: + random_state = None + # + if verbose not in [0, 1, 2]: + raise ValueError('Parameter must be 0, 1, or 2') + # Additional check for inapplicable parameter combinations + # If regression=True we ignore classification-specific parameters and issue user warning + if regression and (needs_proba or stratified): + warn_str = 'This is regression task hence classification-specific parameters set to were ignored:' + if needs_proba: + needs_proba = False + warn_str += ' ' + if stratified: + stratified = False + warn_str += ' ' + warnings.warn(warn_str, UserWarning) + #--------------------------------------------------------------------------- + # Specify default metric + #--------------------------------------------------------------------------- if metric is None and regression: metric = mean_absolute_error elif metric is None and not regression: - metric = accuracy_score - - # Print metric + if needs_proba: + metric = log_loss + else: + metric = accuracy_score + #--------------------------------------------------------------------------- + # Create report header strings and print report header + #--------------------------------------------------------------------------- + if save_dir is not None or verbose > 0: + if regression: + task_str = 'task: [regression]' + else: + task_str = 'task: [classification]' + n_classes_str = 'n_classes: [%d]' % len(np.unique(y_train)) + metric_str = 'metric: [%s]' % metric.__name__ + mode_str = 'mode: [%s]' % mode + n_models_str = 'n_models: [%d]' % len(models) + + # Print report header if verbose > 0: - print('metric: [%s]\n' % metric.__name__) - + print(task_str) + if not regression: + print(n_classes_str) + print(metric_str) + print(mode_str) + print(n_models_str + '\n') + #--------------------------------------------------------------------------- # Split indices to get folds (stratified can be used only for classification) - if stratified and not regression: + #--------------------------------------------------------------------------- + if not regression and stratified: kf = StratifiedKFold(n_splits = n_folds, shuffle = shuffle, random_state = random_state) else: kf = KFold(n_splits = n_folds, shuffle = shuffle, random_state = random_state) - - # Create empty numpy arrays for stacking features - S_train = np.zeros((X_train.shape[0], len(models))) - S_test = np.zeros((X_test.shape[0], len(models))) + #--------------------------------------------------------------------------- + # Compute number of classes (if we need probabilities) to create appropreate empty arrays + # !!! Important. In order to unify array creation variable is always + # equal to 1, except the case when we performing classification task with needs_proba=True + #--------------------------------------------------------------------------- + if not regression and needs_proba: + n_classes = len(np.unique(y_train)) + action = 'predict_proba' + else: + n_classes = 1 + action = 'predict' + #--------------------------------------------------------------------------- + # Create empty numpy arrays for OOF + #--------------------------------------------------------------------------- + if mode in ['oof_pred', 'B', 'oof_pred_bag', 'A']: + S_train = np.zeros(( X_train.shape[0], len(models) * n_classes )) + S_test = np.zeros(( X_test.shape[0], len(models) * n_classes )) + elif mode in ['oof']: + S_train = np.zeros(( X_train.shape[0], len(models) * n_classes )) + S_test = None + elif mode in ['pred', 'pred_bag']: + S_train = None + S_test = np.zeros(( X_test.shape[0], len(models) * n_classes )) + + #--------------------------------------------------------------------------- + # High-level function variables + #--------------------------------------------------------------------------- + # String to store models-folds part of log + models_folds_str = '' + #--------------------------------------------------------------------------- # Loop across models + #--------------------------------------------------------------------------- for model_counter, model in enumerate(models): + if save_dir is not None or verbose > 0: + model_str = 'model %2d: [%s]' % (model_counter, model.__class__.__name__) + if save_dir is not None: + models_folds_str += '-' * 40 + '\n' + models_folds_str += model_str + '\n' + models_folds_str += '-' * 40 + '\n\n' + models_folds_str += model_params(model) if verbose > 0: - print('model %d: [%s]' % (model_counter, model.__class__.__name__)) + print(model_str) # Create empty numpy array, which will contain temporary predictions for test set made in each fold - S_test_temp = np.zeros((X_test.shape[0], n_folds)) + if mode in ['pred_bag', 'oof_pred_bag', 'A']: + S_test_temp = np.zeros((X_test.shape[0], n_folds * n_classes)) + + # Create empty array to store scores for each fold (to find mean) + scores = np.array([]) + #----------------------------------------------------------------------- # Loop across folds - for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): - X_tr = X_train[tr_index] - y_tr = y_train[tr_index] - X_te = X_train[te_index] - y_te = y_train[te_index] - - # Fit 1-st level model - model = model.fit(X_tr, transformer(y_tr, func = transform_target)) - # Predict out-of-fold part of train set - S_train[te_index, model_counter] = transformer(model.predict(X_te), func = transform_pred) - # Predict full test set - S_test_temp[:, fold_counter] = transformer(model.predict(X_test), func = transform_pred) + #----------------------------------------------------------------------- + if mode in ['pred_bag', 'oof', 'oof_pred', 'B', 'oof_pred_bag', 'A']: + for fold_counter, (tr_index, te_index) in enumerate(kf.split(X_train, y_train)): + # Split data and target + X_tr = X_train[tr_index] + y_tr = y_train[tr_index] + X_te = X_train[te_index] + y_te = y_train[te_index] + + # Split sample weights accordingly (if passed) + if sample_weight is not None: + sample_weight_tr = sample_weight[tr_index] + # sample_weight_te = sample_weight[te_index] + else: + sample_weight_tr = None + # sample_weight_te = None + + # Save RAM: clone to avoid fitting model directly inside users list + # Set safe=False to be able to clone non-sklearn models + model = clone(model, safe=False) + + # Fit 1-st level model + if mode in ['pred_bag', 'oof', 'oof_pred', 'B', 'oof_pred_bag', 'A']: + _ = model_action(model, X_tr, y_tr, None, sample_weight = sample_weight_tr, action = 'fit', transform = transform_target) + + # Predict out-of-fold part of train set + if mode in ['oof', 'oof_pred', 'B', 'oof_pred_bag', 'A']: + if 'predict_proba' == action: + col_slice_model = slice(model_counter * n_classes, model_counter * n_classes + n_classes) + else: + col_slice_model = model_counter + S_train[te_index, col_slice_model] = model_action(model, None, None, X_te, action = action, transform = transform_pred) + + # Predict full test set in each fold + if mode in ['pred_bag', 'oof_pred_bag', 'A']: + if 'predict_proba' == action: + col_slice_fold = slice(fold_counter * n_classes, fold_counter * n_classes + n_classes) + else: + col_slice_fold = fold_counter + S_test_temp[:, col_slice_fold] = model_action(model, None, None, X_test, action = action, transform = transform_pred) + + # Compute scores + if mode in ['oof', 'oof_pred', 'B', 'oof_pred_bag', 'A']: + if save_dir is not None or verbose > 0: + score = metric(y_te, S_train[te_index, col_slice_model]) + scores = np.append(scores, score) + fold_str = ' fold %2d: [%.8f]' % (fold_counter, score) + if save_dir is not None: + models_folds_str += fold_str + '\n' + if verbose > 1: + print(fold_str) + + # Compute mean or mode of predictions for test set in bag modes + if mode in ['pred_bag', 'oof_pred_bag', 'A']: + if 'predict_proba' == action: + # Here we copute means of probabilirties for each class + for class_id in range(n_classes): + S_test[:, model_counter * n_classes + class_id] = np.mean(S_test_temp[:, class_id::n_classes], axis = 1) + else: + if regression: + S_test[:, model_counter] = np.mean(S_test_temp, axis = 1) + else: + S_test[:, model_counter] = st.mode(S_test_temp, axis = 1)[0].ravel() - if verbose > 1: - print(' fold %d: [%.8f]' % (fold_counter, metric(y_te, S_train[te_index, model_counter]))) + # Compute scores: mean + std and full + if mode in ['oof', 'oof_pred', 'B', 'oof_pred_bag', 'A']: + if save_dir is not None or verbose > 0: + sep_str = ' ----' + mean_str = ' MEAN: [%.8f] + [%.8f]' % (np.mean(scores), np.std(scores)) + full_str = ' FULL: [%.8f]\n' % (metric(y_train, S_train[:, col_slice_model])) + if save_dir is not None: + models_folds_str += sep_str + '\n' + models_folds_str += mean_str + '\n' + models_folds_str += full_str + '\n' + if verbose > 0: + print(sep_str) + print(mean_str) + print(full_str) - # Compute mean or mode of predictions for test set - if regression: - S_test[:, model_counter] = np.mean(S_test_temp, axis = 1) - else: - S_test[:, model_counter] = st.mode(S_test_temp, axis = 1)[0].ravel() + # Fit model on full train set and predict test set + if mode in ['pred', 'oof_pred', 'B']: + if verbose > 0: + print(' Fitting on full train set...\n') + _ = model_action(model, X_train, y_train, None, sample_weight = sample_weight, action = 'fit', transform = transform_target) + if 'predict_proba' == action: + col_slice_model = slice(model_counter * n_classes, model_counter * n_classes + n_classes) + else: + col_slice_model = model_counter + S_test[:, col_slice_model] = model_action(model, None, None, X_test, action = action, transform = transform_pred) + #--------------------------------------------------------------------------- + # Cast class labels to int + #--------------------------------------------------------------------------- + if not regression and not needs_proba: + if S_train is not None: + S_train = S_train.astype(int) + if S_test is not None: + S_test = S_test.astype(int) + #--------------------------------------------------------------------------- + # Save OOF and log + #--------------------------------------------------------------------------- + if save_dir is not None: + try: + # We have already done save_dir = os.path.normpath(save_dir) - if verbose > 0: - print(' ----') - print(' MEAN: [%.8f]\n' % (metric(y_train, S_train[:, model_counter]))) - + # We generate random last 6 symbols to ensure that name is unique + time_str = datetime.now().strftime('[%Y.%m.%d].[%H.%M.%S].%f') + ('.%06x' % np.random.randint(0xffffff)) + + # Prepare paths for OFF and log files + file_name = time_str + '.npy' + log_file_name = time_str + '.log.txt' + + full_path = os.path.join(save_dir, file_name) + log_full_path = os.path.join(save_dir, log_file_name) + + # Save OOF + array_to_save = np.empty(2, dtype='object') + array_to_save[0] = S_train + array_to_save[1] = S_test + np.save(full_path, array_to_save) + + # Save log + log_str = 'vecstack log ' + log_str += time_str + '\n\n' + log_str += task_str + '\n' + if not regression: + log_str += n_classes_str + '\n' + log_str += metric_str + '\n' + log_str += mode_str + '\n' + log_str += n_models_str + '\n\n' + log_str += models_folds_str + log_str += '-' * 40 + '\n' + log_str += 'END\n' + log_str += '-' * 40 + '\n' + + with open(log_full_path, 'w') as f: + _ = f.write(log_str) + + if verbose > 0: + print('Result was saved to [%s]' % full_path) + except: + print('Error while saving files:\n%s' % sys.exc_info()[1]) + + #--------------------------------------------------------------------------- + # Return + #--------------------------------------------------------------------------- + # For consistency we always return two values: + # 1-st - for train, 2-nd - for test + # Some of this values may be None return (S_train, S_test) #------------------------------------------------------------------------------- diff --git a/vecstack/coresk.py b/vecstack/coresk.py new file mode 100644 index 0000000..f650d2a --- /dev/null +++ b/vecstack/coresk.py @@ -0,0 +1,1144 @@ +"""Scikit-learn compatible API for stacking. + +Find out how to use: +>>> from vecstack import StackingTransformer +>>> help(StackingTransformer) + +MIT License + +Copyright (c) 2016-2025 Igor Ivanov +Email: vecxoz@gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +from __future__ import print_function +from __future__ import division + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +import warnings +from contextlib import suppress +import numpy as np +import scipy.stats as st +from sklearn.base import BaseEstimator +from sklearn.base import TransformerMixin +from sklearn.base import clone +from sklearn.dummy import DummyClassifier +from sklearn.dummy import DummyRegressor +from sklearn.utils.validation import check_X_y +from sklearn.utils.validation import check_array +from sklearn.utils.validation import check_is_fitted +from sklearn.utils.validation import has_fit_parameter +from sklearn.utils.validation import validate_data +from sklearn.model_selection import KFold +from sklearn.model_selection import StratifiedKFold +from sklearn.metrics import mean_absolute_error +from sklearn.metrics import accuracy_score +from sklearn.metrics import log_loss +from sklearn.metrics import mean_squared_error + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + + +class StackingTransformer(TransformerMixin, BaseEstimator): + """StackingTransformer. Scikit-learn compatible API for stacking. + + Parameters + ---------- + estimators : list of tuples, default None + Base level estimators. + If None then by default: + DummyRegressor (predicts constant 5.5) - for regression task + DummyClassifier (predicts constant 1) - for classification task + You can use any sklearn-like estimators. + Each tuple in the list contains arbitrary + unique name and estimator object, e.g.: + estimators = [('lr', LinearRegression()), + ('ridge', Ridge(random_state=0))] + + Note. Dropping individual estimators using 'drop' is not supported. + + Note. According to sklearn convention for binary classification + task with probabilities estimator must return probabilities + for each class (i.e. two columns). + + regression : boolean, default True + If True - perform stacking for regression task, + if False - perform stacking for classification task + + transform_target : callable, default None + Function to transform target variable. + If None - transformation is not used. + For example, for regression task (if target variable is skewed) + you can use transformation like ``numpy.log1p``. + Set ``transform_target=numpy.log1p`` + Usually you want to use respective backward transformation + for prediction like ``numpy.expm1``. + Set ``transform_pred=numpy.expm1`` + Caution! Some transformations may give inapplicable results. + For example, if target variable contains zeros, numpy.log + gives you -inf. In such case you can use appropriate + transformation like ``numpy.log1p`` and respective + backward transformation like ``numpy.expm1`` + + transform_pred : callable, default None + Function to transform prediction. + If None - transformation is not used. + If you use transformation for target variable (``transform_target``) + like ``numpy.log1p``, then using ``transform_pred`` you can specify + respective backward transformation like ``numpy.expm1``. + Look at description of parameter ``transform_target`` + + variant: str, default 'A' + Possible values: 'A', 'B'. + This parameter influences test set predictions only. + Variant 'A' - predict test set in each fold and find mean (mode) + Variant 'B' - fit on full train set and predict test set once + See tutorial for details: + https://github.com/vecxoz/vecstack/blob/master/ \ + examples/00_stacking_concept_pictures_code.ipynb + + needs_proba: boolean, default False, meaningful only for classification + Whether to predict probabilities (instead of class labels) + in classification task. + Ignored if ``regression=True`` + + metric : callable, default None + Evaluation metric (score function) which is used to calculate + cross-validation scores. + If None, then by default: + sklearn.metrics.mean_absolute_error - for regression + sklearn.metrics.accuracy_score - for classification + with class labels + sklearn.metrics.log_loss - for classification with probabilities + You can use any appropriate sklearn metric or + define your own metric like shown below: + + def your_metric(y_true, y_pred): + # calculate + return result + + n_folds : int, default 4 + Number of folds in cross-validation + + stratified : boolean, default False, meaningful only for classification + If True - use stratified folds in cross-validation + Ignored if ``regression=True`` + + shuffle : boolean, default False + Whether to perform a shuffle before cross-validation split + + random_state : int, default 0 + Random seed used to initiate fold split. + Same seed and correspondingly same split is used for all estimators. + Ignored if ``shuffle=False`` + + verbose : int, default 0 + Level of verbosity. + 0 - show no messages + 1 - for each estimator show mean score + 2 - for each estimator show score for each fold and mean score + + + Attributes + ---------- + estimators_ : list + List of base estimators (not fitted) passed by user (or default) + n_estimators_ : int + Number of base estimators passed by user (or default) + n_classes_ : int + Number of classes in classification task. + ``None`` in regression task. + models_A_ : list of lists + List containing n_estimators lists. Each of which contains + n_folds models (fitted estimators). + In variant A this models are used to transform (predict) + both train set and test set + In variant B this models are used to transform (predict) + train set only + models_B_ : list or None + List containing n_estimators models (fitted estimators). + In variant A this attribute is None + In variant B this models are used to transform (predict) test set + metric_ : callable + Metric passed by user (or default) which was used + to compute cross-validation scores + kf_ : KFold or StratifiedKFold object + Initialized cross-validation object which was used to split train set + during fitting + scores_ : 2d numpy array of shape [n_estimators, n_folds] + Scores for each fold for each estimator. + mean_std_ : list of tuples + Each tuple contains name, mean and std for each estimator. + train_shape_ : tuple of ints + Shape of training data + n_train_examples_ : int + Number of training examples + n_features_: int + Number of features + train_footprint_ : list of tuples + Train set footprint which is used to identify train set + during transform (predict) phase. + Each tuple containes 3 values: row index, column index, data value + + Examples + -------- + >>> from sklearn.datasets import fetch_california_housing + >>> from sklearn.model_selection import train_test_split + >>> from sklearn.metrics import mean_absolute_error + >>> from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor + >>> from xgboost import XGBRegressor + >>> from vecstack import StackingTransformer + >>> + >>> # Load demo data + >>> X, y = fetch_california_housing(return_X_y=True) + >>> + >>> # Make train/test split + >>> X_train, X_test, y_train, y_test = train_test_split(X, y, + test_size=0.2, + random_state=0) + >>> + >>> # Init 1st level estimators + >>> estimators_L1 = [('et', ExtraTreesRegressor(random_state=0, + n_jobs=-1, + n_estimators=100, + max_depth=3)), + ('rf', RandomForestRegressor(random_state=0, + n_jobs=-1, + n_estimators=100, + max_depth=3)), + ('xgb', XGBRegressor(random_state=0, + n_jobs=-1, + learning_rate=0.1, + n_estimators=100, + max_depth=3))] + >>> # Stacking + >>> stack = StackingTransformer(estimators=estimators_L1, + regression=True, + shuffle=True, + random_state=0, + verbose=2) + >>> stack = stack.fit(X_train, y_train) + >>> S_train = stack.transform(X_train) + >>> S_test = stack.transform(X_test) + >>> + >>> # Use 2nd level estimator to get final prediction + >>> estimator_L2 = XGBRegressor(random_state=0, + n_jobs=-1, + learning_rate=0.1, + n_estimators=100, + max_depth=3) + >>> estimator_L2 = estimator_L2.fit(S_train, y_train) + >>> y_pred = estimator_L2.predict(S_test) + >>> + >>> # Final prediction score + >>> print('Final score: [%.8f]' % mean_absolute_error(y_test, y_pred)) + """ + def __init__(self, + estimators=None, + regression=True, + transform_target=None, + transform_pred=None, + variant='A', + needs_proba=False, + metric=None, + n_folds=4, + stratified=False, + shuffle=False, + random_state=0, + verbose=0): + + self.estimators = estimators + self.regression = regression + self.transform_target = transform_target + self.transform_pred = transform_pred + self.variant = variant + self.needs_proba = needs_proba + self.metric = metric + self.n_folds = n_folds + self.stratified = stratified + self.shuffle = shuffle + self.random_state = random_state + self.verbose = verbose + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def __sklearn_tags__(self): + tags = super().__sklearn_tags__() + tags.estimator_type = 'transformer' + tags.transformer_tags.preserves_dtype = [] + tags.target_tags.required = True + tags.input_tags.sparse = True + return tags + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def fit(self, X, y, sample_weight=None): + """Fit all base estimators. + + Parameters + ---------- + X : 2d numpy array or sparse matrix of shape [n_samples, n_features] + Training data + y : 1d numpy array of shape [n_samples] + Target values. + sample_weight : 1d numpy array of shape [n_samples] + Individual weights for each sample. + Passed to fit method of each estimator. + Note: will be split automatically for each fold. + + Returns + ------- + self : object + Fitted StackingTransformer instance. + """ + # --------------------------------------------------------------------- + # Validation + # --------------------------------------------------------------------- + + # --------------------------------------------------------------------- + # Check input data + # --------------------------------------------------------------------- + # Check data and set `self.n_features_in_` and `self.feature_names_in_` + X, y = validate_data(self, X, y, + reset=True, # default: True, if True will set `self.n_features_in_` and `self.feature_names_in_` + validate_separately=False, # default: False, if False will use `check_X_y` + skip_check_array=False, # default: False, if False will NOT skip checks + accept_sparse=['csr'], + ensure_all_finite=True, + multi_output=False) + + # Legacy check included in `validate_data` + # Check X and y + # ``check_estimator`` does not allow ``ensure_all_finite=False`` + # X, y = check_X_y(X, y, + # accept_sparse=['csr'], # allow csr, cast all others to csr + # ensure_all_finite=True, # do not allow nan and inf + # multi_output=False) # allow only one column in y_train + + # Check X and sample_weight + # X is alredy checked, but we need it to compare length of sample_weight + if sample_weight is not None: + X, sample_weight = check_X_y(X, sample_weight, + accept_sparse=['csr'], + ensure_all_finite=True, + multi_output=False) + + # --------------------------------------------------------------------- + # Check ``estimators`` + # --------------------------------------------------------------------- + if self.estimators is None: + if self.regression: + self.estimators_ = [('dumregr', DummyRegressor(strategy='constant', constant=5.5))] + else: + self.estimators_ = [('dumclf', DummyClassifier(strategy='constant', constant=1))] + # warnings.warn('No estimators were specified. ' + # 'Using single dummy estimator as demo.', UserWarning) + else: + if 0 == len(self.estimators): + raise ValueError('List of estimators is empty') + else: + # Clone + self.estimators_ = [(name, clone(estim)) for name, estim in self.estimators] + # Check names of estimators + names, estims = zip(*self.estimators_) + self._validate_names(names) + # Check if all estimators support ``sample_weight`` + if sample_weight is not None: + for name, estim in self.estimators_: + if not has_fit_parameter(estim, 'sample_weight'): + raise ValueError('Underlying estimator [%s] does not ' + 'support sample weights.' % name) + + # --------------------------------------------------------------------- + # Check other StackingTransformer parameters + # --------------------------------------------------------------------- + + # ``variant`` + if self.variant not in ['A', 'B']: + raise ValueError('Parameter ``variant`` must be set properly') + + # ``n_folds`` + if not isinstance(self.n_folds, int): + raise ValueError('Parameter ``n_folds`` must be integer') + if not self.n_folds > 1: + raise ValueError('Parameter ``n_folds`` must be not less than 2') + + # ``verbose`` + if self.verbose not in [0, 1, 2]: + raise ValueError('Parameter ``verbose`` must be 0, 1, or 2') + + # Additional check for inapplicable parameter combinations + # If ``regression=True`` we ignore classification-specific + # parameters and issue user warning + if self.regression and (self.needs_proba or self.stratified): + warn_str = ('This is regression task hence classification-specific' + 'parameters set to ``True`` were ignored:') + if self.needs_proba: + self.needs_proba = False + warn_str += ' ``needs_proba``' + if self.stratified: + self.stratified = False + warn_str += ' ``stratified``' + warnings.warn(warn_str, UserWarning) + + # To comply with sklearn requirement + if not self.shuffle: + random_state_internal = None + else: + random_state_internal = self.random_state + + # --------------------------------------------------------------------- + # Compute attributes (basic properties of data, number of estimators, etc.) + # --------------------------------------------------------------------- + self.train_shape_ = X.shape + self.n_train_examples_ = X.shape[0] + self.n_features_ = X.shape[1] + if not self.regression: + self.n_classes_ = len(np.unique(y)) + else: + self.n_classes_ = None + self.n_estimators_ = len(self.estimators_) + self.train_footprint_ = self._get_footprint(X) + + # --------------------------------------------------------------------- + # Specify default metric + # --------------------------------------------------------------------- + if self.metric is None and self.regression: + self.metric_ = mean_absolute_error + elif self.metric is None and not self.regression: + if self.needs_proba: + self.metric_ = log_loss + else: + self.metric_ = accuracy_score + else: + self.metric_ = self.metric + # --------------------------------------------------------------------- + # Create report header strings and print report header + # --------------------------------------------------------------------- + if self.verbose > 0: + if self.regression: + task_str = 'task: [regression]' + else: + task_str = 'task: [classification]' + n_classes_str = 'n_classes: [%d]' % self.n_classes_ + metric_str = 'metric: [%s]' % self.metric_.__name__ + variant_str = 'variant: [%s]' % self.variant + n_estimators_str = 'n_estimators: [%d]' % self.n_estimators_ + + print(task_str) + if not self.regression: + print(n_classes_str) + print(metric_str) + print(variant_str) + print(n_estimators_str + '\n') + # --------------------------------------------------------------------- + # Initialize cross-validation split + # Stratified can be used only for classification + # --------------------------------------------------------------------- + if not self.regression and self.stratified: + self.kf_ = StratifiedKFold(n_splits=self.n_folds, + shuffle=self.shuffle, + random_state=random_state_internal) + # Save target to be able to create stratified split in ``transform`` method + # This is more efficient than to save split indices + self._y_ = y.copy() + else: + self.kf_ = KFold(n_splits=self.n_folds, + shuffle=self.shuffle, + random_state=random_state_internal) + self._y_ = None + + # --------------------------------------------------------------------- + # Compute implicit number of classes to create appropriate empty arrays. + # !!! Important. In order to unify array creation + # variable ``n_classes_implicit_`` is always equal to 1, except the case + # when we performing classification task with ``needs_proba=True`` + # --------------------------------------------------------------------- + if not self.regression and self.needs_proba: + self.n_classes_implicit_ = len(np.unique(y)) + self.action_ = 'predict_proba' + else: + self.n_classes_implicit_ = 1 + self.action_ = 'predict' + + # --------------------------------------------------------------------- + # Create empty numpy array for train predictions (OOF) + # !!! Important. We have to implicitly predict during fit + # in order to compute CV scores, because + # the most reasonable place to print out CV scores is fit method + # --------------------------------------------------------------------- + S_train = np.zeros((X.shape[0], self.n_estimators_ * self.n_classes_implicit_)) + + # --------------------------------------------------------------------- + # Prepare (clone) estmators for fitting and storing + # We need models_A_ for both variant A and varian B + # We need models_B_ for varian B only (in variant A attribute models_B_ is None) + # --------------------------------------------------------------------- + + self.models_A_ = [] + self.models_B_ = None + + for n, est in self.estimators_: + self.models_A_.append([clone(est) for _ in range(self.n_folds)]) + + if self.variant in ['B']: + self.models_B_ = [clone(est) for n, est in self.estimators_] + + # --------------------------------------------------------------------- + # Create empty numpy array to store scores for each estimator and each fold + # --------------------------------------------------------------------- + self.scores_ = np.zeros((self.n_estimators_, self.n_folds)) + + # --------------------------------------------------------------------- + # Create empty list to store name, mean and std for each estimator + # --------------------------------------------------------------------- + self.mean_std_ = [] + + # --------------------------------------------------------------------- + # MAIN FIT PROCEDURE + # --------------------------------------------------------------------- + # Loop across estimators + # --------------------------------------------------------------------- + for estimator_counter, (name, estimator) in enumerate(self.estimators_): + if self.verbose > 0: + estimator_str = 'estimator %2d: [%s: %s]' % (estimator_counter, name, estimator.__class__.__name__) + print(estimator_str) + + # ----------------------------------------------------------------- + # Loop across folds + # ----------------------------------------------------------------- + for fold_counter, (tr_index, te_index) in enumerate(self.kf_.split(X, y)): + # Split data and target + X_tr = X[tr_index] + y_tr = y[tr_index] + X_te = X[te_index] + y_te = y[te_index] + + # Split sample weights accordingly (if passed) + if sample_weight is not None: + sample_weight_tr = sample_weight[tr_index] + # sample_weight_te = sample_weight[te_index] + else: + sample_weight_tr = None + # sample_weight_te = None + + # Fit estimator + _ = self._estimator_action(self.models_A_[estimator_counter][fold_counter], + X_tr, y_tr, None, + sample_weight=sample_weight_tr, + action='fit', + transform=self.transform_target) + + # Predict out-of-fold part of train set + if 'predict_proba' == self.action_: + col_slice_estimator = slice(estimator_counter * self.n_classes_implicit_, + estimator_counter * self.n_classes_implicit_ + self.n_classes_implicit_) + else: + col_slice_estimator = estimator_counter + S_train[te_index, col_slice_estimator] = self._estimator_action(self.models_A_[estimator_counter][fold_counter], + None, None, + X_te, action=self.action_, + transform=self.transform_pred) + # Compute score + score = self.metric_(y_te, S_train[te_index, col_slice_estimator]) + self.scores_[estimator_counter, fold_counter] = score + + # Print fold score + if self.verbose > 1: + fold_str = ' fold %2d: [%.8f]' % (fold_counter, score) + print(fold_str) + + # Compute mean and std and save in dict + estim_name = self.estimators_[estimator_counter][0] + estim_mean = np.mean(self.scores_[estimator_counter]) + estim_std = np.std(self.scores_[estimator_counter]) + self.mean_std_.append((estim_name, estim_mean, estim_std)) + + if self.verbose > 1: + sep_str = ' ----' + print(sep_str) + + # Compute mean + std (and full) + if self.verbose > 0: + mean_str = ' MEAN: [%.8f] + [%.8f]\n' % (estim_mean, estim_std) + print(mean_str) + + # Fit estimator on full train set + if self.variant in ['B']: + if self.verbose > 0: + print(' Fitting on full train set...\n') + _ = self._estimator_action(self.models_B_[estimator_counter], + X, y, None, + sample_weight=sample_weight, + action='fit', + transform=self.transform_target) + + # --------------------------------------------------------------------- + # --------------------------------------------------------------------- + + # Return fitted StackingTransformer instance + return self + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def fit_transform(self, X, y, sample_weight=None): + """Fit all base estimators and transform (predict) train set. + + Parameters + ---------- + See docs for ``fit`` and ``transform`` methods. + + Returns + ------- + X_transformed : 2d numpy array of shape [n_samples, n_estimators] or + [n_samples, n_estimators * n_classes] + Out-of-fold predictions (OOF) for train set. + This is stacked features for next level. + """ + # --------------------------------------------------------------------- + # All validation and procedures are done inside corresponding methods + # fit and transform + # --------------------------------------------------------------------- + + return self.fit(X, y, sample_weight).transform(X) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def transform(self, X, is_train_set=None): + """Transform (predict) given data set. + If ``X`` is train set: + for each estimator return out-of-fold predictions (OOF). + If ``X`` is any other set: + variant A: for each estimator return mean (mode) of predictions + made in each fold + variant B: for each estimator return single prediction + + Parameters + ---------- + X : 2d numpy array or sparse matrix of shape [n_samples, n_features] + Input data + + is_train_set : boolean, default None + Fallback parameter. In general case + should not be used (should be None). + Gives ability to explicitly specify that given dataset + is train set or other set. + + Returns + ------- + X_transformed : 2d numpy array of shape [n_samples, n_estimators] or + [n_samples, n_estimators * n_classes] + Out-of-fold predictions (OOF) for train set. + Regular or bagged predictions for any other set. + This is stacked features for next level. + """ + # Check if fitted + check_is_fitted(self, ['models_A_']) + + # Check data without resetting `self.n_features_in_` and `self.feature_names_in_` + X = validate_data(self, X, + reset=False, # default: True, if True will set `self.n_features_in_` and `self.feature_names_in_` + validate_separately=False, # default: False, if False will use `check_X_y` + skip_check_array=False, # default: False, if False will NOT skip checks + accept_sparse=['csr'], + ensure_all_finite=True) # no need for `multi_output`, because no `y` + + # Legacy check included in `validate_data` + # Input validation + # ``check_estimator`` does not allow ``ensure_all_finite=False`` + # X = check_array(X, accept_sparse=['csr'], ensure_all_finite=True) + + # ********************************************************************* + # Fitted StackingTransformer instance is bound to train set used for fitting. + # So during transformation we have different actions for train set + # and all other sets + # ********************************************************************* + + if is_train_set is None: + is_train_set = self._check_identity(X) + + # Print + if self.verbose > 0: + if is_train_set: + print('Train set was detected.') + print('Transforming...\n') + + # ********************************************************************* + # Transform train set + # ********************************************************************* + if is_train_set: + + # In case if user directly tells that it is train set but shape is different + if self.train_shape_ != X.shape: + raise ValueError('Train set must have the same shape ' + 'in order to be transformed.') + + # Create empty numpy array for train predictions (OOF) + S_train = np.zeros((X.shape[0], self.n_estimators_ * self.n_classes_implicit_)) + + # ----------------------------------------------------------------- + # MAIN TRANSFORM (PREDICT) PROCEDURE for train set + # ----------------------------------------------------------------- + # Loop across estimators + # ----------------------------------------------------------------- + for estimator_counter, (name, estimator) in enumerate(self.estimators_): + if self.verbose > 0: + estimator_str = 'estimator %2d: [%s: %s]' % (estimator_counter, name, estimator.__class__.__name__) + print(estimator_str) + + # ------------------------------------------------------------- + # Loop across folds + # ------------------------------------------------------------- + for fold_counter, (tr_index, te_index) in enumerate(self.kf_.split(X, self._y_)): + # Split data + # X_tr = X[tr_index] + X_te = X[te_index] + + # Predict out-of-fold part of train set + if 'predict_proba' == self.action_: + col_slice_estimator = slice(estimator_counter * self.n_classes_implicit_, + estimator_counter * self.n_classes_implicit_ + self.n_classes_implicit_) + else: + col_slice_estimator = estimator_counter + S_train[te_index, col_slice_estimator] = self._estimator_action(self.models_A_[estimator_counter][fold_counter], + None, None, + X_te, action=self.action_, + transform=self.transform_pred) + # Print + if self.verbose > 1: + fold_str = ' model from fold %2d: done' % fold_counter + print(fold_str) + + if self.verbose > 1: + sep_str = ' ----' + print(sep_str) + + if self.verbose > 0: + done_str = ' DONE\n' + print(done_str) + + # ----------------------------------------------------------------- + # Cast class labels to int + # ----------------------------------------------------------------- + if not self.regression and not self.needs_proba: + S_train = S_train.astype(int) + + # Return transformed data (OOF) + return S_train # X_transformed + + # ********************************************************************* + # Transform any other set + # ********************************************************************* + else: + # Legacy check included in `validate_data` + # Check n_features + # if X.shape[1] != self.n_features_: + # raise ValueError('Inconsistent number of features.') + + # Create empty numpy array for test predictions + S_test = np.zeros((X.shape[0], self.n_estimators_ * self.n_classes_implicit_)) + + # --------------------------------------------------------------------- + # MAIN TRANSFORM (PREDICT) PROCEDURE for any other set + # ----------------------------------------------------------------- + # Loop across estimators + # ----------------------------------------------------------------- + for estimator_counter, (name, estimator) in enumerate(self.estimators_): + if self.verbose > 0: + estimator_str = 'estimator %2d: [%s: %s]' % (estimator_counter, name, estimator.__class__.__name__) + print(estimator_str) + # ------------------------------------------------------------- + # Variant A + # ------------------------------------------------------------- + if self.variant in ['A']: + # Create empty numpy array, which will contain temporary predictions + # for test set made in each fold + S_test_temp = np.zeros((X.shape[0], self.n_folds * self.n_classes_implicit_)) + # --------------------------------------------------------- + # Loop across fitted models (it is the same as loop across folds) + # --------------------------------------------------------- + for fold_counter, model in enumerate(self.models_A_[estimator_counter]): + # Predict test set in each fold + if 'predict_proba' == self.action_: + col_slice_fold = slice(fold_counter * self.n_classes_implicit_, + fold_counter * self.n_classes_implicit_ + self.n_classes_implicit_) + else: + col_slice_fold = fold_counter + S_test_temp[:, col_slice_fold] = self._estimator_action(model, None, None, X, + action=self.action_, + transform=self.transform_pred) + # Print + if self.verbose > 1: + fold_str = ' model from fold %2d: done' % fold_counter + print(fold_str) + + if self.verbose > 1: + sep_str = ' ----' + print(sep_str) + + # --------------------------------------------------------- + # Compute mean or mode (majority voting) of predictions for test set + # --------------------------------------------------------- + if 'predict_proba' == self.action_: + # Here we copute means of probabilirties for each class + for class_id in range(self.n_classes_implicit_): + S_test[:, estimator_counter * self.n_classes_implicit_ + class_id] = np.mean(S_test_temp[:, class_id::self.n_classes_implicit_], axis=1) + else: + if self.regression: + S_test[:, estimator_counter] = np.mean(S_test_temp, axis=1) + else: + S_test[:, estimator_counter] = st.mode(S_test_temp, axis=1)[0].ravel() + + if self.verbose > 0: + done_str = ' DONE\n' + print(done_str) + + # ------------------------------------------------------------- + # Variant B + # ------------------------------------------------------------- + else: + if 'predict_proba' == self.action_: + col_slice_estimator = slice(estimator_counter * self.n_classes_implicit_, + estimator_counter * self.n_classes_implicit_ + self.n_classes_implicit_) + else: + col_slice_estimator = estimator_counter + S_test[:, col_slice_estimator] = self._estimator_action(self.models_B_[estimator_counter], + None, None, X, + action=self.action_, + transform=self.transform_pred) + + if self.verbose > 0: + done_str = ' DONE\n' + print(done_str) + + # --------------------------------------------------------------------- + # Cast class labels to int + # --------------------------------------------------------------------- + if not self.regression and not self.needs_proba: + S_test = S_test.astype(int) + + return S_test # X_transformed + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _transformer(self, y, func=None): + """Transforms target variable and prediction + """ + if func is None: + return y + else: + return func(y) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _estimator_action(self, estimator, X_train, y_train, X_test, + sample_weight=None, action=None, + transform=None): + """Performs estimator action. + This wrapper gives us ability to choose action dynamically + (e.g. ``predict`` or ``predict_proba``). + Note. Through ``_estimator_action`` and then through ``_transformer`` + we apply ``transform_target`` and ``transform_pred`` functions if + given by user on the target and prediction in each fold separately + to be able to calculate proper scores. + """ + if 'fit' == action: + # We use following condition, because some estimators (e.g. Lars) + # may not have ``sample_weight`` parameter of ``fit`` method + if sample_weight is not None: + return estimator.fit(X_train, self._transformer(y_train, func=transform), + sample_weight=sample_weight) + else: + return estimator.fit(X_train, self._transformer(y_train, func=transform)) + elif 'predict' == action: + return self._transformer(estimator.predict(X_test), func=transform) + elif 'predict_proba' == action: + return self._transformer(estimator.predict_proba(X_test), func=transform) + else: + raise ValueError('Parameter action must be set properly') + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _random_choice(self, n, size, bound=2**30): + """ + Memory efficient substitute for np.random.choice without replacement + + Parameters: + =========== + n : int + Upper value for range to chose from: [0, n). + This parameter is bounded (see bound). + size: int + Number of values to chose + bound : int + Upper random int for backward compatibility + with some older numpy versions + + Returns: + ======== + ids : 1d numpy array of shape (size, ) dtype=np.int32 + """ + try: + if n < size: + raise ValueError('Drawing without replacement: ' + '``n`` cannot be less than ``size``') + + ids = [] + while len(ids) < size: + rnd = np.random.randint(min(bound, n)) + if rnd not in ids: + ids.append(rnd) + return np.array(ids, dtype=np.int32) + + except Exception: + raise ValueError('Internal error. ' + 'Please save traceback and inform developers.') + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _get_footprint(self, X, n_items=1000): + """Selects ``n_items`` random elements from 2d numpy array or + sparse matrix (or all elements if their number is less or equal + to ``n_items``). + """ + try: + footprint = [] + r, c = X.shape + n = r * c + # np.random.seed(0) # for development + + # OOM with large arrays (see #29) + # ids = np.random.choice(n, min(n_items, n), replace=False) + + ids = self._random_choice(n, min(n_items, n)) + + for i in ids: + row = i // c + col = i - row * c + footprint.append((row, col, X[row, col])) + + return footprint + + except Exception: + raise ValueError('Internal error. ' + 'Please save traceback and inform developers.') + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _check_identity(self, X, + rtol=1e-05, atol=1e-08, + equal_nan=False): + """Checks 2d numpy array or sparse matrix identity + by its shape and footprint. + """ + try: + # Check shape + if X.shape != self.train_shape_: + return False + # Check footprint + try: + for coo in self.train_footprint_: + assert np.isclose(X[coo[0], coo[1]], coo[2], rtol=rtol, atol=atol, equal_nan=equal_nan) + return True + except AssertionError: + return False + + except Exception: + raise ValueError('Internal error. ' + 'Please save traceback and inform developers.') + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _get_params(self, attr, deep=True): + """ + Gives ability to get parameters of nested estimators + """ + out = super().get_params(deep=deep) + if not deep: + return out + + estimators = getattr(self, attr) + try: + out.update(estimators) + except (TypeError, ValueError): + # Ignore TypeError for cases where estimators is not a list of + # (name, estimator) and ignore ValueError when the list is not + # formatted correctly. This is to prevent errors when calling + # `set_params`. `BaseEstimator.set_params` calls `get_params` which + # can error for invalid values for `estimators`. + return out + + for name, estimator in estimators: + if hasattr(estimator, 'get_params'): + for key, value in estimator.get_params(deep=True).items(): + out['%s__%s' % (name, key)] = value + return out + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def get_params(self, deep=True): + """ + Get parameters of StackingTransformer and base estimators. + + Parameters + ---------- + deep : boolean + If False - get parameters of StackingTransformer + If True - get parameters of StackingTransformer and base estimators + + Returns + ------- + params : dict + Parameter and estimator names mapped to their values or parameter + names mapped to their values. + """ + return self._get_params('estimators', deep=deep) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _set_params(self, attr, **params): + """ + Gives ability to set parameters of nested estimators, + and replace individual estimators in the list. + """ + # Ensure strict ordering of parameter setting: + # 1. Replace the entire estimators collection + if attr in params: + setattr(self, attr, params.pop(attr)) + # 2. Replace individual estimators by name + items = getattr(self, attr) + if isinstance(items, list) and items: + # Get item names used to identify valid names in params + # `zip` raises a TypeError when `items` does not contains + # elements of length 2 + with suppress(TypeError): + item_names, _ = zip(*items) + for name in list(params.keys()): + if '__' not in name and name in item_names: + self._replace_estimator(attr, name, params.pop(name)) + + # 3. Individual estimator parameters and other initialisation arguments + super().set_params(**params) + return self + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _replace_estimator(self, attr, name, new_val): + """ + Replace estimator, assuming `name` is a valid estimator name + """ + new_estimators = list(getattr(self, attr)) + for i, (estimator_name, _) in enumerate(new_estimators): + if estimator_name == name: + new_estimators[i] = (name, new_val) + break + setattr(self, attr, new_estimators) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def set_params(self, **params): + """ + Set parameters of StackingTransformer and base estimators. + Valid parameter keys can be listed with `get_params()`. Note that you + can directly set the parameters of the estimators contained in `estimators`. + + Parameters + ---------- + **params : keyword arguments + Specific parameters using e.g. `set_params(parameter_name=new_value)`. + In addition, to setting the parameters of the estimator, + the individual estimator of the estimators can also be set. + Dropping individual estimators using 'drop' is not supported. + + Returns + ------- + self : object + Estimator instance. + """ + self._set_params('estimators', **params) + return self + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def _validate_names(self, names): + """Validates estimator names + """ + if len(set(names)) != len(names): + raise ValueError('Names provided are not unique: ' + '%s' % list(names)) + invalid_names = set(names).intersection(self.get_params(deep=False)) + if invalid_names: + raise ValueError('Estimator names conflict with constructor ' + 'arguments: %s' % sorted(invalid_names)) + invalid_names = [name for name in names if '__' in name] + if invalid_names: + raise ValueError('Estimator names must not contain __: got ' + '%s' % invalid_names) + + # ------------------------------------------------------------------------- + # ------------------------------------------------------------------------- + + def is_train_set(self, X): + """Checks if given data set was used to train + StackingTransformer instance. + + Parameters + ---------- + X : 2d numpy array or sparse matrix of shape [n_samples, n_features] + Input data + + Returns + ------- + check_result : boolean + True - if X was used to train StackingTransformer instance + False - otherwise + """ + # Check if fitted + check_is_fitted(self, ['models_A_']) + # Input validation + X = check_array(X, accept_sparse=['csr'], ensure_all_finite=True) + return self._check_identity(X) + +# ----------------------------------------------------------------------------- +# -----------------------------------------------------------------------------