diff --git a/docs/readthedocs/image/friesian_architecture.png b/docs/readthedocs/image/friesian_architecture.png new file mode 100644 index 00000000..b0b3f14e Binary files /dev/null and b/docs/readthedocs/image/friesian_architecture.png differ diff --git a/docs/readthedocs/image/scala_logo.png b/docs/readthedocs/image/scala_logo.png new file mode 100644 index 00000000..193034df Binary files /dev/null and b/docs/readthedocs/image/scala_logo.png differ diff --git a/docs/readthedocs/requirements-doc.txt b/docs/readthedocs/requirements-doc.txt index f88098b9..aa36b82d 100644 --- a/docs/readthedocs/requirements-doc.txt +++ b/docs/readthedocs/requirements-doc.txt @@ -11,9 +11,9 @@ cloudpickle==2.1.0 ray[tune]==1.9.2 ray==1.9.2 torch==1.9.0 -Pygments==2.6.1 +Pygments==2.7 setuptools==41.0.1 -docutils==0.17 +docutils==0.17.1 mock==1.0.1 pillow==5.4.1 sphinx==4.5.0 @@ -21,7 +21,6 @@ alabaster>=0.7,<0.8,!=0.7.5 commonmark==0.8.1 recommonmark==0.5.0 readthedocs-sphinx-ext<2.2 -sphinx_rtd_theme==1.0.0 scikit-learn==1.0.2 pystan==2.19.1.1 prophet==1.0.1 @@ -40,4 +39,5 @@ sphinx-external-toc==0.3.0 nbsphinx==0.8.9 ipython==7.34.0 sphinx-design==0.2.0 -nbsphinx-link==1.3.0 \ No newline at end of file +nbsphinx-link==1.3.0 +pydata-sphinx-theme==0.11.0 \ No newline at end of file diff --git a/docs/readthedocs/source/_static/css/chronos_installation_guide.css b/docs/readthedocs/source/_static/css/chronos_installation_guide.css index 8bfc2412..11229176 100644 --- a/docs/readthedocs/source/_static/css/chronos_installation_guide.css +++ b/docs/readthedocs/source/_static/css/chronos_installation_guide.css @@ -11,7 +11,7 @@ } #table-1 tr, td{ - background-color: rgb(240, 241, 245); + background-color: var(--pst-color-on-surface); height: 30px; border-width: 2px; border-style: solid; @@ -26,7 +26,7 @@ #table-1 td{ font-size: 16px; font-family: Verdana; - color: rgb(15, 24, 33); + color: var(--pst-color-text-base); text-align: center; /* height: 56px; line-height: 56px; */ diff --git a/docs/readthedocs/source/_static/css/custom.css b/docs/readthedocs/source/_static/css/custom.css index 023bf856..3f105261 100644 --- a/docs/readthedocs/source/_static/css/custom.css +++ b/docs/readthedocs/source/_static/css/custom.css @@ -1,65 +1,63 @@ -/*Extends the docstring signature box.*/ -.rst-content dl:not(.docutils) dt { - display: block; - padding: 10px; - word-wrap: break-word; - padding-right: 100px; -} -/*Lists in an admonition note do not have awkward whitespace below.*/ -.rst-content .admonition-note .section ul { - margin-bottom: 0px; -} -/*Properties become blue (classmethod, staticmethod, property)*/ -.rst-content dl dt em.property { - color: #2980b9; - text-transform: uppercase; +/* change primary & info color for light mode*/ +html[data-theme="light"] { + --pst-color-primary: rgb(1, 113, 195); + --pst-color-info: rgb(1, 113, 195); } -.rst-content .section ol p, -.rst-content .section ul p { - margin-bottom: 0px; +/* ectra css variables */ +:root { + --pst-color-info-tiny-opacity: rgba(1, 113, 195, 0.1); + --pst-color-info-low-opacity: rgba(1, 113, 195, 0.25); } -div.sphx-glr-bigcontainer { - display: inline-block; - width: 100%; + +/* align items in the left part of header to the ground*/ +.bd-header #navbar-start { + align-items: end; } -td.tune-colab, -th.tune-colab { - border: 1px solid #dddddd; - text-align: left; - padding: 8px; +/* for version badge, possible for other badges*/ +.version-badge{ + border: 1px solid var(--pst-color-primary); + border-radius: 0.25rem; + color: var(--pst-color-primary); + padding: 0.1rem 0.25rem; + font-size: var(--pst-font-size-milli); } -/* Adjustment to Sphinx Book Theme */ -.table td { - /* Remove row spacing */ - padding: 0; +/* for card components */ +.bd-content .sd-card { + border: none; + border-left: .2rem solid var(--pst-color-info-low-opacity); } -table { - /* Force full width for all table */ - width: 136% !important; +.bd-content .sd-card .sd-card-header{ + background-color: var(--pst-color-info-tiny-opacity); + border: none; } -img.inline-figure { - /* Override the display: block for img */ - display: inherit !important; +.bigdl-link-card:hover{ + border-left: .2rem solid var(--pst-color-info); } -#version-warning-banner { - /* Make version warning clickable */ - z-index: 1; +/* for sphinx-design badge components (customized for usage in card footer)*/ +.sd-badge{ + padding: .35em 0em; + font-size: 0.9em; } -span.rst-current-version > span.fa.fa-book { - /* Move the book icon away from the top right - * corner of the version flyout menu */ - margin: 10px 0px 0px 5px; +/* for landing page side bar */ +.bigdl-quicklinks-section-nav{ + padding-bottom: 0.5rem; + padding-left: 1rem; } -/* Adjustment to Version block */ -.rst-versions { - z-index: 1200 !important; +.bigdl-quicklinks-section-title{ + color: var(--pst-color-primary); } + +/* force long parameter definition (which occupy a whole line) + to break in api documents for class/method */ +.sig-object{ + overflow-wrap: break-word; +} \ No newline at end of file diff --git a/docs/readthedocs/source/_static/js/chronos_installation_guide.js b/docs/readthedocs/source/_static/js/chronos_installation_guide.js index b9fbae35..5709217e 100644 --- a/docs/readthedocs/source/_static/js/chronos_installation_guide.js +++ b/docs/readthedocs/source/_static/js/chronos_installation_guide.js @@ -232,8 +232,8 @@ function refresh_cmd(){ //set the color of selected buttons function set_color(id){ - $("#"+id).parent().css("background-color","rgb(74, 106, 237)"); - $("#"+id).css("color","white"); + $("#"+id).parent().css("background-color","var(--pst-color-primary)"); + $("#"+id).css("color","var(--pst-color-primary-text)"); $("#"+id).addClass("isset"); } @@ -241,7 +241,7 @@ function set_color(id){ function reset_color(list){ for (btn in list){ $("#"+list[btn]).parent().css("background-color","transparent"); - $("#"+list[btn]).css("color","black"); + $("#"+list[btn]).css("color","var(--pst-color-text-base)"); $("#"+list[btn]).removeClass("isset"); } } @@ -254,7 +254,7 @@ function disable(list){ } reset_color(list); for(btn in list){ - $("#"+list[btn]).parent().css("background-color","rgb(133, 133, 133)"); + $("#"+list[btn]).parent().css("background-color","var(--pst-color-muted)"); } } @@ -303,14 +303,14 @@ $(document).on('click',"button",function(){ $(document).on({ mouseenter: function () { if($(this).prop("disabled")!=true){ - $(this).parent().css("background-color","rgb(74, 106, 237)"); - $(this).css("color","white"); + $(this).parent().css("background-color","var(--pst-color-primary)"); + $(this).css("color","var(--pst-color-primary-text)"); } }, mouseleave: function () { if(!$(this).hasClass("isset") && $(this).prop("disabled")!=true){ $(this).parent().css("background-color","transparent"); - $(this).css("color","black"); + $(this).css("color","var(--pst-color-text-base)"); } } }, "button"); diff --git a/docs/readthedocs/source/_static/js/chronos_tutorial.js b/docs/readthedocs/source/_static/js/chronos_tutorial.js index 1c66daf1..39abda71 100644 --- a/docs/readthedocs/source/_static/js/chronos_tutorial.js +++ b/docs/readthedocs/source/_static/js/chronos_tutorial.js @@ -24,8 +24,9 @@ function disCheck(ids){ //event when click the checkboxes $(".checkboxes").click(function(){ //get all checked values + //class checkboxes is specified to avoid selecting toctree checkboxes (arrows) var vals = []; - $('input:checkbox:checked').each(function (index, item) { + $('.checkboxes:input:checkbox:checked').each(function (index, item) { vals.push($(this).val()); }); diff --git a/docs/readthedocs/source/_static/js/custom.js b/docs/readthedocs/source/_static/js/custom.js new file mode 100644 index 00000000..ab7d3019 --- /dev/null +++ b/docs/readthedocs/source/_static/js/custom.js @@ -0,0 +1,26 @@ +$(document).ready(function(){ +// $('.btn.dropdown-toggle.nav-item').text('Libraries'); // change text for dropdown menu in header from More to Libraries + + // hide the original left sidebar ads display + $('#ethical-ad-placement').css({ + "display":"none" + }); + + // manually add the ads to the end of content + $(".bd-article").append( + "
\ +
\ + " + ); + + // make tables responsive + $("table").wrap( + "
" + ); +}) \ No newline at end of file diff --git a/docs/readthedocs/source/_templates/breadcrumbs.html b/docs/readthedocs/source/_templates/breadcrumbs.html deleted file mode 100644 index 599388dc..00000000 --- a/docs/readthedocs/source/_templates/breadcrumbs.html +++ /dev/null @@ -1,60 +0,0 @@ - - -{%- extends "sphinx_rtd_theme/breadcrumbs.html" %} - - -{%- block breadcrumbs_aside %} -
  • - {%- if hasdoc(pagename) and display_vcs_links %} - {%- if display_github %} - {%- if check_meta and 'github_url' in meta %} - - {{ _('Edit this page') }} - {%- else %} - {{ _('Edit this page') }} - {%- endif %} - {%- elif show_source and source_url_prefix %} - {{ _('View page source') }} - {%- elif show_source and has_source and sourcename %} - {{ _('View page source') }} - {%- endif %} - {%- endif %} -
    • -{%- endblock %} \ No newline at end of file diff --git a/docs/readthedocs/source/_templates/sidebar_backbutton.html b/docs/readthedocs/source/_templates/sidebar_backbutton.html new file mode 100644 index 00000000..fc2482c8 --- /dev/null +++ b/docs/readthedocs/source/_templates/sidebar_backbutton.html @@ -0,0 +1,6 @@ +{% set home_href = pathto(master_doc) %} +
    + + Back to Homepage ↵ + +
    \ No newline at end of file diff --git a/docs/readthedocs/source/_templates/sidebar_quicklinks.html b/docs/readthedocs/source/_templates/sidebar_quicklinks.html new file mode 100644 index 00000000..a781b61d --- /dev/null +++ b/docs/readthedocs/source/_templates/sidebar_quicklinks.html @@ -0,0 +1,68 @@ + \ No newline at end of file diff --git a/docs/readthedocs/source/_templates/version_badge.html b/docs/readthedocs/source/_templates/version_badge.html new file mode 100644 index 00000000..8eeaa61f --- /dev/null +++ b/docs/readthedocs/source/_templates/version_badge.html @@ -0,0 +1,3 @@ +
    + {{ release }} +
    \ No newline at end of file diff --git a/docs/readthedocs/source/_toc.yml b/docs/readthedocs/source/_toc.yml index d91eb186..7c05e4dc 100644 --- a/docs/readthedocs/source/_toc.yml +++ b/docs/readthedocs/source/_toc.yml @@ -1,105 +1,224 @@ root: index subtrees: - - caption: Quick Start - entries: - - file: doc/Orca/QuickStart/orca-tf-quickstart - - file: doc/Orca/QuickStart/orca-keras-quickstart - - file: doc/Orca/QuickStart/orca-tf2keras-quickstart - - file: doc/Orca/QuickStart/orca-pytorch-quickstart - - file: doc/Ray/QuickStart/ray-quickstart + - entries: + - file: doc/UserGuide/index + title: 'User guide' + subtrees: + - entries: + - file: doc/UserGuide/python + - file: doc/UserGuide/scala + - file: doc/UserGuide/win + - file: doc/UserGuide/docker + - file: doc/UserGuide/colab + - file: doc/UserGuide/hadoop + - file: doc/UserGuide/k8s + - file: doc/UserGuide/databricks - - caption: User Guide - entries: - - file: doc/UserGuide/python - - file: doc/UserGuide/scala - - file: doc/UserGuide/win - - file: doc/UserGuide/colab - - file: doc/UserGuide/docker - - file: doc/UserGuide/hadoop - - file: doc/UserGuide/k8s - - file: doc/UserGuide/databricks - - file: doc/UserGuide/develop - - file: doc/UserGuide/known_issues - - caption: Nano - entries: - - file: doc/Nano/Overview/nano - - file: doc/Nano/QuickStart/pytorch_train - - file: doc/Nano/QuickStart/pytorch_inference - - file: doc/Nano/QuickStart/tensorflow_train - - file: doc/Nano/QuickStart/tensorflow_inference - - file: doc/Nano/QuickStart/hpo - - file: doc/Nano/QuickStart/index - - file: doc/Nano/Howto/index - - file: doc/Nano/Overview/known_issues + - entries: + - file: doc/Application/powered-by + title: "Powered by" - - caption: DLlib - entries: - - file: doc/DLlib/Overview/dllib - - file: doc/DLlib/Overview/keras-api - - file: doc/DLlib/Overview/nnframes - - caption: Orca - entries: - - file: doc/Orca/Overview/orca - title: "Orca User Guide" - - file: doc/Orca/Overview/orca-context - - file: doc/Orca/Overview/data-parallel-processing - - file: doc/Orca/Overview/distributed-training-inference - - file: doc/Orca/Overview/distributed-tuning - - file: doc/Ray/Overview/ray - - file: doc/Orca/Overview/known_issues + - entries: + - file: doc/Orca/index + title: "Orca" + subtrees: + - entries: + - file: doc/Orca/Overview/orca + title: "Orca in 5 miniutes" + - file: doc/Orca/Overview/install + title: "Installation" + - file: doc/Orca/Overview/index + title: "Key Features" + subtrees: + - entries: + - file: doc/Orca/Overview/orca-context + - file: doc/Orca/Overview/data-parallel-processing + - file: doc/Orca/Overview/distributed-training-inference + - file: doc/Orca/Overview/distributed-tuning + - file: doc/Orca/Overview/ray + - file: doc/Orca/QuickStart/index + title: "Tutorials" + subtrees: + - entries: + - file: doc/UseCase/spark-dataframe + - file: doc/UseCase/xshards-pandas + - file: doc/Orca/QuickStart/ray-quickstart + - file: doc/Orca/QuickStart/orca-pytorch-distributed-quickstart + - file: doc/Orca/QuickStart/orca-autoestimator-pytorch-quickstart + - file: doc/Orca/QuickStart/orca-autoxgboost-quickstart + - file: doc/Orca/Overview/known_issues + title: "Tips and Known Issues" + - file: doc/PythonAPI/Orca/index + title: "API Reference" - - caption: Chronos - entries: - - file: doc/Chronos/Overview/chronos - - file: doc/Chronos/Overview/quick-tour - - file: doc/Chronos/Howto/index - - file: doc/Chronos/QuickStart/index - - file: doc/Chronos/Overview/deep_dive - - file: doc/Chronos/Overview/chronos_known_issue - - caption: PPML - entries: - - file: doc/PPML/Overview/ppml - - file: doc/PPML/Overview/trusted_big_data_analytics_and_ml - - file: doc/PPML/Overview/trusted_fl - - file: doc/PPML/QuickStart/secure_your_services - - file: doc/PPML/QuickStart/build_kernel_with_sgx - - file: doc/PPML/QuickStart/deploy_intel_sgx_device_plugin_for_kubernetes - - file: doc/PPML/QuickStart/trusted-serving-on-k8s-guide - - file: doc/PPML/QuickStart/tpc-h_with_sparksql_on_k8s - - file: doc/PPML/QuickStart/tpc-ds_with_sparksql_on_k8s - - file: doc/PPML/Overview/azure_ppml - - caption: Serving - entries: - - file: doc/Serving/Overview/serving.md - - file: doc/Serving/QuickStart/serving-quickstart - - file: doc/Serving/ProgrammingGuide/serving-installation - - file: doc/Serving/ProgrammingGuide/serving-start - - file: doc/Serving/ProgrammingGuide/serving-inference - - file: doc/Serving/Example/example - - file: doc/Serving/FAQ/faq - - file: doc/Serving/FAQ/contribute-guide + - entries: + - file: doc/Nano/index + title: "Nano" + subtrees: + - entries: + - file: doc/Nano/Overview/nano + title: "Nano in 5 minutes" + - file: doc/Nano/Overview/install + title: "Installation" + - file: doc/Nano/Overview/index + title: "Key Features" + subtrees: + - entries: + - file: doc/Nano/Overview/pytorch_train + - file: doc/Nano/Overview/pytorch_inference + - file: doc/Nano/Overview/tensorflow_train + - file: doc/Nano/Overview/tensorflow_inference + - file: doc/Nano/Overview/hpo + - file: doc/Nano/QuickStart/index + title: "Tutorials" + subtrees: + - entries: + - file: doc/Nano/QuickStart/pytorch_train_quickstart + - file: doc/Nano/QuickStart/pytorch_onnxruntime + - file: doc/Nano/QuickStart/pytorch_openvino + - file: doc/Nano/QuickStart/pytorch_quantization_inc_onnx + - file: doc/Nano/QuickStart/pytorch_quantization_inc + - file: doc/Nano/QuickStart/pytorch_quantization_openvino + - file: doc/Nano/QuickStart/tensorflow_train_quickstart + - file: doc/Nano/QuickStart/tensorflow_embedding + - file: doc/Nano/QuickStart/tensorflow_quantization_quickstart + - file: doc/Nano/Howto/index + title: "How-to Guides" + - file: doc/Nano/Overview/known_issues + title: "Tips and Known Issues" + - file: doc/PythonAPI/Nano/index + title: "API Reference" - - caption: Common Use Case - entries: - - file: doc/Orca/QuickStart/orca-pytorch-distributed-quickstart - - file: doc/UseCase/spark-dataframe - - file: doc/UseCase/xshards-pandas - - file: doc/Orca/QuickStart/orca-autoestimator-pytorch-quickstart - - file: doc/Orca/QuickStart/orca-autoxgboost-quickstart - - caption: Python API - entries: - - file: doc/PythonAPI/Orca/orca - - file: doc/PythonAPI/Friesian/feature - - file: doc/PythonAPI/Chronos/index - - file: doc/PythonAPI/Nano/index - - caption: Real-World Application - entries: - - file: doc/Application/presentations - - file: doc/Application/blogs - - file: doc/Application/powered-by + - entries: + - file: doc/DLlib/index + title: "DLlib" + subtrees: + - entries: + - file: doc/DLlib/Overview/dllib + title: "DLLib in 5 minutes" + - file: doc/DLlib/Overview/install + title: "Installation" + - file: doc/DLlib/Overview/index + title: "Key Features" + subtrees: + - entries: + - file: doc/DLlib/Overview/keras-api + - file: doc/DLlib/Overview/nnframes + - file: doc/DLlib/Overview/visualization + title: "Visualization" + - file: doc/DLlib/QuickStart/index + title: "Tutorials" + subtrees: + - entries: + - file: doc/DLlib/QuickStart/python-getting-started + title: "Python Quick Start" + - file: doc/DLlib/QuickStart/scala-getting-started + title: "Scala Quick Start" + - file: doc/PythonAPI/DLlib/index + title: "API Reference" + + - entries: + - file: doc/Chronos/index + title: "Chronos" + subtrees: + - entries: + - file: doc/Chronos/Overview/quick-tour + title: "Chronos in 5 minutes" + - file: doc/Chronos/Overview/install + title: "Installation" + - file: doc/Chronos/Overview/deep_dive + title: "Key Features" + - file: doc/Chronos/Howto/index + title: "How-to Guides" + - file: doc/Chronos/QuickStart/index + title: "Tutorials" + subtrees: + - entries: + - file: doc/Chronos/QuickStart/chronos-tsdataset-forecaster-quickstart + - file: doc/Chronos/QuickStart/chronos-autotsest-quickstart + - file: doc/Chronos/QuickStart/chronos-anomaly-detector + - file: doc/Chronos/Overview/chronos_known_issue + title: "Tips and Known Issues" + - file: doc/PythonAPI/Chronos/index + title: "API Reference" + + - entries: + - file: doc/Friesian/index + title: "Friesian" + subtrees: + - entries: + - file: doc/Friesian/intro + title: "Introduction" + - file: doc/Friesian/serving + title: "Serving" + - file: doc/Friesian/examples + title: "Use Cases" + - file: doc/PythonAPI/Friesian/index + title: "API Reference" + + - entries: + - file: doc/PPML/index + title: "PPML" + subtrees: + - entries: + - file: doc/PPML/Overview/intro + title: "PPML Introduction" + - file: doc/PPML/Overview/userguide + title: 'User Guide' + - file: doc/PPML/Overview/examples + title: "Tutorials" + subtrees: + - entries: + - file: doc/PPML/Overview/quicktour + - file: doc/PPML/QuickStart/end-to-end + - file: doc/PPML/Overview/misc + title: "Advanced Topics" + subtrees: + - entries: + - file: doc/PPML/Overview/ppml + - file: doc/PPML/Overview/trusted_big_data_analytics_and_ml + - file: doc/PPML/Overview/trusted_fl + - file: doc/PPML/QuickStart/secure_your_services + - file: doc/PPML/QuickStart/build_kernel_with_sgx + - file: doc/PPML/QuickStart/deploy_intel_sgx_device_plugin_for_kubernetes + - file: doc/PPML/QuickStart/trusted-serving-on-k8s-guide + - file: doc/PPML/QuickStart/tpc-h_with_sparksql_on_k8s + - file: doc/PPML/QuickStart/tpc-ds_with_sparksql_on_k8s + - file: doc/PPML/Overview/azure_ppml + + + - entries: + - file: doc/UserGuide/develop + title: "Developer guide" + + + - entries: + - file: doc/Serving/index + title: "Cluster serving" + subtrees: + - entries: + - file: doc/Serving/Overview/serving.md + title: "User Guide" + - file: doc/Serving/QuickStart/serving-quickstart + title: "Serving in 5 miniutes" + - file: doc/Serving/ProgrammingGuide/serving-installation + - file: doc/Serving/ProgrammingGuide/serving-start + - file: doc/Serving/ProgrammingGuide/serving-inference + - file: doc/Serving/Example/example + title: "Examples" + - file: doc/Serving/FAQ/faq + - file: doc/Serving/FAQ/contribute-guide + + + - entries: + - file: doc/Application/presentations + title: "Presentations" + + - entries: + - file: doc/Application/blogs diff --git a/docs/readthedocs/source/conf.py b/docs/readthedocs/source/conf.py index e111cdbc..b274fbac 100644 --- a/docs/readthedocs/source/conf.py +++ b/docs/readthedocs/source/conf.py @@ -31,19 +31,39 @@ sys.path.insert(0, os.path.abspath("../../../python/serving/src/")) sys.path.insert(0, os.path.abspath("../../../python/nano/src/")) # -- Project information ----------------------------------------------------- -import sphinx_rtd_theme -html_theme = "sphinx_rtd_theme" -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -#html_theme = "sphinx_book_theme" +html_theme = "pydata_sphinx_theme" html_theme_options = { - "repository_url": "https://github.com/intel-analytics/BigDL", - "use_repository_button": True, - "use_issues_button": True, - "use_edit_page_button": True, - "path_to_docs": "doc/source", - "home_page_in_toc": True, + "header_links_before_dropdown": 8, + "icon_links": [ + { + "name": "GitHub Repository for BigDL", + "url": "https://github.com/intel-analytics/BigDL", + "icon": "fa-brands fa-square-github", + "type": "fontawesome", + } + ], + "navbar_start": ["navbar-logo.html", "version_badge.html"], + "navbar_end": ["navbar-icon-links.html"], # remove dark mode for now } +# add search bar to side bar +html_sidebars = { + "index": [ + "sidebar_quicklinks.html" + ], + "**": ["sidebar_backbutton.html","sidebar-nav-bs.html"] +} + +# remove dark mode for now +html_context = { + "default_mode": "light" +} + +html_logo = "../image/bigdl_logo.png" + +# hard code it for now, may change it to read from installed bigdl +release = "latest" + # The suffix of source filenames. from recommonmark.parser import CommonMarkParser source_suffix = {'.rst': 'restructuredtext', @@ -92,7 +112,8 @@ extensions = [ 'sphinx_external_toc', 'sphinx_design', 'nbsphinx', - 'nbsphinx_link' + 'nbsphinx_link', + 'sphinx.ext.graphviz' # for embedded graphviz diagram ] # Add any paths that contain templates here, relative to this directory. @@ -136,6 +157,13 @@ exclude_patterns = ['_build'] # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +# add js/css for customizing each page +html_js_files = [ + 'js/custom.js', +] +html_css_files = [ + 'css/custom.css', +] # Custom sidebar templates, must be a dictionary that maps document namesan # to template names. @@ -246,4 +274,7 @@ def setup(app): app.add_transform(AutoStructify) # disable notebook execution -nbsphinx_execute = 'never' \ No newline at end of file +nbsphinx_execute = 'never' + +# make output of graphviz diagram to svg +graphviz_output_format = 'svg' \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Application/index.rst b/docs/readthedocs/source/doc/Application/index.rst new file mode 100644 index 00000000..7ec694eb --- /dev/null +++ b/docs/readthedocs/source/doc/Application/index.rst @@ -0,0 +1,2 @@ +Real-World Application +========================= \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Chronos/Howto/docker_guide_single_node.md b/docs/readthedocs/source/doc/Chronos/Howto/docker_guide_single_node.md index 0922747d..164327b7 100644 --- a/docs/readthedocs/source/doc/Chronos/Howto/docker_guide_single_node.md +++ b/docs/readthedocs/source/doc/Chronos/Howto/docker_guide_single_node.md @@ -97,15 +97,15 @@ After the Jupyter Notebook service is successfully started, you can connect to t You should shut down the BigDL Docker container after using it. 1. First, use `ctrl+p+q` to quit the container when you are still in it. 2. Then, you can list all the active Docker containers by command line: -```bash -sudo docker ps -``` -You will see your docker containers: -```bash -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -40de2cdad025 chronos-nightly:b1 "/opt/work/" 3 hours ago Up 3 hours upbeat_al -``` + ```bash + sudo docker ps + ``` + You will see your docker containers: + ```bash + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 40de2cdad025 chronos-nightly:b1 "/opt/work/" 3 hours ago Up 3 hours upbeat_al + ``` 3. Shut down the corresponding docker container by its ID: -```bash -sudo docker rm -f 40de2cdad025 -``` + ```bash + sudo docker rm -f 40de2cdad025 + ``` diff --git a/docs/readthedocs/source/doc/Chronos/Image/anomaly_detection.svg b/docs/readthedocs/source/doc/Chronos/Image/anomaly_detection.svg index 643aba2a..41c488ca 100644 --- a/docs/readthedocs/source/doc/Chronos/Image/anomaly_detection.svg +++ b/docs/readthedocs/source/doc/Chronos/Image/anomaly_detection.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Chronos/Image/forecasting.svg b/docs/readthedocs/source/doc/Chronos/Image/forecasting.svg index b35dfe3c..7d1fc66d 100644 --- a/docs/readthedocs/source/doc/Chronos/Image/forecasting.svg +++ b/docs/readthedocs/source/doc/Chronos/Image/forecasting.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Chronos/Image/simulation.svg b/docs/readthedocs/source/doc/Chronos/Image/simulation.svg index 1d8bf61f..71744d3a 100644 --- a/docs/readthedocs/source/doc/Chronos/Image/simulation.svg +++ b/docs/readthedocs/source/doc/Chronos/Image/simulation.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Chronos/Overview/anomaly_detection.md b/docs/readthedocs/source/doc/Chronos/Overview/anomaly_detection.md index 7c27b9c9..c021c6dc 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/anomaly_detection.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/anomaly_detection.md @@ -1,29 +1,29 @@ -# Time Series Anomaly Detection Overview +# Anomaly Detection -Anomaly Detection detects abnormal samples in a given time series. _Chronos_ provides a set of unsupervised anomaly detectors. +Anomaly Detection detects abnormal samples in a given time series. _Chronos_ provides a set of unsupervised anomaly detectors. View some examples notebooks for [Datacenter AIOps][AIOps]. ## **1. ThresholdDetector** -ThresholdDetector detects anomaly based on threshold. It can be used to detect anomaly on a given time series ([notebook][AIOps_anomaly_detect_unsupervised]), or used together with [Forecasters](#forecasting) to detect anomaly on new coming samples ([notebook][AIOps_anomaly_detect_unsupervised_forecast_based]). +ThresholdDetector detects anomaly based on threshold. It can be used to detect anomaly on a given time series ([notebook][AIOps_anomaly_detect_unsupervised]), or used together with [Forecasters](#forecasting) to detect anomaly on new coming samples ([notebook][AIOps_anomaly_detect_unsupervised_forecast_based]). View [ThresholdDetector API Doc](../../PythonAPI/Chronos/anomaly_detectors.html#chronos-model-anomaly-th-detector) for more details. ## **2. AEDetector** -AEDetector detects anomaly based on the reconstruction error of an autoencoder network. +AEDetector detects anomaly based on the reconstruction error of an autoencoder network. View anomaly detection [notebook][AIOps_anomaly_detect_unsupervised] and [AEDetector API Doc](../../PythonAPI/Chronos/anomaly_detectors.html#chronos-model-anomaly-ae-detector) for more details. ## **3. DBScanDetector** -DBScanDetector uses DBSCAN clustering algortihm for anomaly detection. +DBScanDetector uses DBSCAN clustering algortihm for anomaly detection. ```eval_rst -.. note:: - Users may install `scikit-learn-intelex` to accelerate this detector. Chronos will detect if `scikit-learn-intelex` is installed to decide if using it. More details please refer to: https://intel.github.io/scikit-learn-intelex/installation.html +.. note:: + Users may install ``scikit-learn-intelex`` to accelerate this detector. Chronos will detect if ``scikit-learn-intelex`` is installed to decide if using it. More details please refer to: https://intel.github.io/scikit-learn-intelex/installation.html ``` View anomaly detection [notebook][AIOps_anomaly_detect_unsupervised] and [DBScanDetector API Doc](../../PythonAPI/Chronos/anomaly_detectors.html#chronos-model-anomaly-dbscan-detector) for more details. diff --git a/docs/readthedocs/source/doc/Chronos/Overview/data_processing_feature_engineering.md b/docs/readthedocs/source/doc/Chronos/Overview/data_processing_feature_engineering.md index 13ab1b51..5d21a0ca 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/data_processing_feature_engineering.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/data_processing_feature_engineering.md @@ -1,4 +1,4 @@ -# Time Series Processing and Feature Engineering Overview +# Data Processing and Feature Engineering Time series data is a special data formulation with its specific operations. _Chronos_ provides [`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) as a time series dataset abstract for data processing (e.g. impute, deduplicate, resample, scale/unscale, roll sampling) and auto feature engineering (e.g. datetime feature, aggregation feature). Chronos also provides [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html#xshardstsdataset) with same(or similar) API for distributed and parallelized data preprocessing on large data. @@ -6,7 +6,7 @@ Users can create a [`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) quickly ## **1. Basic concepts** -A time series can be interpreted as a sequence of real value whose order is timestamp. While a time series dataset can be a combination of one or a huge amount of time series. It may contain multiple time series since users may collect different time series in the same/different period of time (e.g. An AIops dataset may have CPU usage ratio and memory usage ratio data for two servers at a period of time. This dataset contains four time series). +A time series can be interpreted as a sequence of real value whose order is timestamp. While a time series dataset can be a combination of one or a huge amount of time series. It may contain multiple time series since users may collect different time series in the same/different period of time (e.g. An AIops dataset may have CPU usage ratio and memory usage ratio data for two servers at a period of time. This dataset contains four time series). In [`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) and [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html#xshardstsdataset), we provide **2** possible dimensions to construct a high dimension time series dataset (i.e. **feature dimension** and **id dimension**). @@ -16,10 +16,10 @@ In [`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) and [`XShardsTSDataset` All the preprocessing operations will be done on each independent time series(i.e on both feature dimension and id dimension), while feature scaling will be only carried out on the feature dimension. ```eval_rst -.. note:: - +.. note:: + ``XShardsTSDataset`` will perform the data processing in parallel(based on spark) to support large dataset. While the parallelization will only be performed on "id dimension". This means, in previous example, ``XShardsTSDataset`` will only utilize multiple workers to process data for different servers at the same time. If a dataset only has 1 id, ``XShardsTSDataset`` will be even slower than ``TSDataset`` because of the overhead. - + ``` ## **2. Create a TSDataset** @@ -40,13 +40,13 @@ You can initialize a [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html .. code-block:: python # Server id Datetime CPU usage Mem usage - # 0 08:39 2021/7/9 93 24 - # 0 08:40 2021/7/9 91 24 - # 0 08:41 2021/7/9 93 25 + # 0 08:39 2021/7/9 93 24 + # 0 08:40 2021/7/9 91 24 + # 0 08:41 2021/7/9 93 25 # 0 ... ... ... - # 1 08:39 2021/7/9 73 79 - # 1 08:40 2021/7/9 72 80 - # 1 08:41 2021/7/9 79 80 + # 1 08:39 2021/7/9 73 79 + # 1 08:40 2021/7/9 72 80 + # 1 08:41 2021/7/9 79 80 # 1 ... ... ... from bigdl.chronos.data import TSDataset @@ -74,14 +74,14 @@ You can initialize a [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html target_col="value", id_col="id", extra_feature_col=["extra feature 1", "extra feature 2"]) - + ``` `target_col` is a list of all elements along feature dimension, while `id_col` is the identifier that distinguishes the id dimension. `dt_col` is the datetime column. For `extra_feature_col`(not shown in this case), you should list those features that you are not interested for your task (e.g. you will **not** perform forecasting or anomaly detection task on this col). If you are building a prototype for your forecasting/anomaly detection task and you need to split you TSDataset to train/valid/test set, you can use `with_split` parameter.[`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) or [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html#xshardstsdataset) supports split with ratio by `val_ratio` and `test_ratio`. ## **3. Time series dataset preprocessing** -[`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) supports [`impute`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.impute), [`deduplicate`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.deduplicate) and [`resample`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.resample). You may fill the missing point by [`impute`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.impute) in different modes. You may remove the records that are totally the same by [`deduplicate`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.deduplicate). You may change the sample frequency by [`resample`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.resample). [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html#xshardstsdataset) only supports [`impute`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.experimental.xshards_tsdataset.XShardsTSDataset.impute) for now. +[`TSDataset`](../../PythonAPI/Chronos/tsdataset.html) supports [`impute`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.impute), [`deduplicate`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.deduplicate) and [`resample`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.resample). You may fill the missing point by [`impute`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.impute) in different modes. You may remove the records that are totally the same by [`deduplicate`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.deduplicate). You may change the sample frequency by [`resample`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.resample). [`XShardsTSDataset`](../../PythonAPI/Chronos/tsdataset.html#xshardstsdataset) only supports [`impute`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.experimental.xshards_tsdataset.XShardsTSDataset.impute) for now. A typical cascade call for preprocessing is: ```eval_rst @@ -92,7 +92,7 @@ A typical cascade call for preprocessing is: .. code-block:: python tsdata.deduplicate().resample(interval="2s").impute() - + .. tab:: XShardsTSDataset .. code-block:: python @@ -109,7 +109,7 @@ Since a scaler should not fit, a typical call for scaling operations is is: .. tabs:: .. tab:: TSDataset - + .. code-block:: python from sklearn.preprocessing import StandardScaler @@ -139,14 +139,14 @@ Since a scaler should not fit, a typical call for scaling operations is is: for tsdata in [tsdata_train, tsdata_valid, tsdata_test]: tsdata.unscale() ``` -[`unscale_numpy`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.unscale_numpy) in TSDataset or [`unscale_xshards`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.experimental.xshards_tsdataset.XShardsTSDataset.unscale_xshards) in XShardsTSDataset is specially designed for forecasters. Users may unscale the output of a forecaster by this operation. +[`unscale_numpy`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.unscale_numpy) in TSDataset or [`unscale_xshards`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.experimental.xshards_tsdataset.XShardsTSDataset.unscale_xshards) in XShardsTSDataset is specially designed for forecasters. Users may unscale the output of a forecaster by this operation. A typical call is: ```eval_rst .. tabs:: .. tab:: TSDataset - + .. code-block:: python x, y = tsdata_test.scale(scaler)\ @@ -156,9 +156,9 @@ A typical call is: unscaled_yhat = tsdata_test.unscale_numpy(yhat) unscaled_y = tsdata_test.unscale_numpy(y) # calculate metric by unscaled_yhat and unscaled_y - + .. tab:: XShardsTSDataset - + .. code-block:: python x, y = tsdata_test.scale(scaler)\ @@ -176,28 +176,28 @@ Other than historical target data and other extra feature provided by users, som A time series dataset needs to be sampling and exporting as numpy ndarray/dataloader to be used in machine learning and deep learning models(e.g. forecasters, anomaly detectors, auto models, etc.). ```eval_rst .. warning:: - You don't need to call any sampling or exporting methods introduced in this section when using `AutoTSEstimator`. + You don't need to call any sampling or exporting methods introduced in this section when using ``AutoTSEstimator``. ``` ### **6.1 Roll sampling** -Roll sampling (or sliding window sampling) is useful when you want to train a RR type supervised deep learning forecasting model. It works as the [diagram](#RR-forecast-image) shows. +Roll sampling (or sliding window sampling) is useful when you want to train a RR type supervised deep learning forecasting model. It works as the [diagram](#RR-forecast-image) shows. Please refer to the API doc [`roll`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.roll) for detailed behavior. Users can simply export the sampling result as numpy ndarray by [`to_numpy`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.to_numpy), pytorch dataloader [`to_torch_data_loader`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.to_torch_data_loader), tensorflow dataset by [to_tf_dataset](https://bigdl.readthedocs.io/en/latest/doc/PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.tsdataset.TSDataset.to_tf_dataset) or xshards object by [to_xshards](https://bigdl.readthedocs.io/en/latest/doc/PythonAPI/Chronos/tsdataset.html#bigdl.chronos.data.experimental.xshards_tsdataset.XShardsTSDataset.to_xshards). ```eval_rst -.. note:: - **Difference between `roll` and `to_torch_data_loader`**: - - `.roll(...)` performs the rolling before RR forecasters/auto models training while `.to_torch_data_loader(...)` performs rolling during the training. - - It is fine to use either of them when you have a relatively small dataset (less than 1G). `.to_torch_data_loader(...)` is recommended when you have a large dataset (larger than 1G) to save memory usage. +.. note:: + **Difference between** ``roll`` **and** ``to_torch_data_loader``: + + ``.roll(...)`` performs the rolling before RR forecasters/auto models training while ``.to_torch_data_loader(...)`` performs rolling during the training. + + It is fine to use either of them when you have a relatively small dataset (less than 1G). ``.to_torch_data_loader(...)`` is recommended when you have a large dataset (larger than 1G) to save memory usage. ``` ```eval_rst -.. note:: +.. note:: **Roll sampling format**: - + As decribed in RR style forecasting concept, the sampling result will have the following shape requirement. | x: (sample_num, lookback, input_feature_num) @@ -218,7 +218,7 @@ A typical call of [`roll`](../../PythonAPI/Chronos/tsdataset.html#bigdl.chronos. # forecaster x, y = tsdata.roll(lookback=..., horizon=...).to_numpy() forecaster.fit((x, y)) - + .. tab:: XShardsTSDataset .. code-block:: python @@ -235,7 +235,7 @@ Now we support pandas dataframe exporting through `to_pandas()` for users to car x = tsdata.to_pandas()["target"].to_numpy() anomaly_detector.fit(x) ``` -View [TSDataset API Doc](../../PythonAPI/Chronos/tsdataset.html#) for more details. +View [TSDataset API Doc](../../PythonAPI/Chronos/tsdataset.html#) for more details. ## **7. Built-in Dataset** diff --git a/docs/readthedocs/source/doc/Chronos/Overview/forecasting.md b/docs/readthedocs/source/doc/Chronos/Overview/forecasting.md index ad6d8b9f..ba1f5236 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/forecasting.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/forecasting.md @@ -1,4 +1,4 @@ -# Time Series Forecasting Overview +# Time Series Forecasting _Chronos_ provides both deep learning/machine learning models and traditional statistical models for forecasting. @@ -67,16 +67,16 @@ For AutoTS Pipeline, we will leverage `AutoTSEstimator`, `TSPipeline` and prefer 3. Use the returned `TSPipeline` for further development. ```eval_rst .. warning:: - `AutoTSTrainer` workflow has been deprecated, no feature updates or performance improvement will be carried out. Users of `AutoTSTrainer` may refer to `Chronos API doc `_. + ``AutoTSTrainer`` workflow has been deprecated, no feature updates or performance improvement will be carried out. Users of ``AutoTSTrainer`` may refer to `Chronos API doc `_. ``` ```eval_rst .. note:: - `AutoTSEstimator` currently only support pytorch backend. + ``AutoTSEstimator`` currently only support pytorch backend. ``` View [Quick Start](../QuickStart/chronos-autotsest-quickstart.html) for a more detailed example. ##### **2.1 Prepare dataset** -`AutoTSEstimator` support 2 types of data input. +`AutoTSEstimator` support 2 types of data input. You can easily prepare your data in `TSDataset` (recommended). You may refer to [here](#TSDataset) for the detailed information to prepare your `TSDataset` with proper data processing and feature generation. Here is a typical `TSDataset` preparation. ```python @@ -107,7 +107,7 @@ auto_estimator = AutoTSEstimator(model='lstm', search_space='normal', past_seq_len=hp.randint(1, 10), future_seq_len=1, - selected_features="auto") + selected_features="auto") ``` We prebuild three defualt search space for each build-in model, which you can use the by setting `search_space` to "minimal"，"normal", or "large" or define your own search space in a dictionary. The larger the search space, the better accuracy you will get and the more time will be cost. @@ -147,7 +147,7 @@ Detailed information please refer to [TSPipeline API doc](../../PythonAPI/Chrono ```eval_rst .. note:: - `init_orca_context` is not needed if you just use the trained TSPipeline for inference, evaluation or incremental fitting. + ``init_orca_context`` is not needed if you just use the trained TSPipeline for inference, evaluation or incremental fitting. ``` ```eval_rst .. note:: @@ -160,7 +160,7 @@ _Chronos_ provides a set of standalone time series forecasters without AutoML su View some examples notebooks for [Network Traffic Prediction][network_traffic] -The common process of using a Forecaster looks like below. +The common process of using a Forecaster looks like below. ```python # set fixed hyperparameters, loss, metric... f = Forecaster(...) @@ -197,9 +197,9 @@ View Network Traffic multivariate multistep Prediction [notebook][network_traffi ##### **3.4 MTNetForecaster** ```eval_rst -.. note:: +.. note:: **Additional Dependencies**: - You need to install `bigdl-nano[tensorflow]` to enable this built-in model. + You need to install ``bigdl-nano[tensorflow]`` to enable this built-in model. ``pip install bigdl-nano[tensorflow]`` ``` @@ -219,9 +219,9 @@ View High-dimensional Electricity Data Forecasting [example][run_electricity] an ##### **3.6 ARIMAForecaster** ```eval_rst -.. note:: +.. note:: **Additional Dependencies**: - You need to install `pmdarima` to enable this built-in model. + You need to install ``pmdarima`` to enable this built-in model. ``pip install pmdarima==1.8.5`` ``` @@ -234,7 +234,7 @@ View [ARIMAForecaster API Doc](../../PythonAPI/Chronos/forecasters.html#arimafor ##### **3.7 ProphetForecaster** ```eval_rst -.. note:: +.. note:: **Additional Dependencies**: You need to install `prophet` to enable this built-in model. @@ -242,7 +242,7 @@ View [ARIMAForecaster API Doc](../../PythonAPI/Chronos/forecasters.html#arimafor ``` ```eval_rst -.. note:: +.. note:: **Acceleration Note**: Intel® Distribution for Python may improve the speed of prophet's training and inferencing. You may install it by refering to https://www.intel.com/content/www/us/en/developer/tools/oneapi/distribution-for-python.html. ``` diff --git a/docs/readthedocs/source/doc/Chronos/Overview/chronos.md b/docs/readthedocs/source/doc/Chronos/Overview/install.md similarity index 59% rename from docs/readthedocs/source/doc/Chronos/Overview/chronos.md rename to docs/readthedocs/source/doc/Chronos/Overview/install.md index 67d8bf68..3705f2af 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/chronos.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/install.md @@ -1,59 +1,30 @@ -# Chronos User Guide - -### **1. Overview** -_BigDL-Chronos_ (_Chronos_ for short) is an application framework for building a fast, accurate and scalable time series analysis application. - -You can use _Chronos_ to: - -```eval_rst -.. grid:: 3 - :gutter: 1 - - .. grid-item-card:: - :class-footer: sd-bg-light - - **Forecasting** - ^^^ - - .. image:: ../Image/forecasting.svg - :width: 200 - :alt: Alternative text - - +++ - - Predict future using history data. - - .. grid-item-card:: - :class-footer: sd-bg-light - - **Anomaly Detection** - ^^^ - - .. image:: ../Image/anomaly_detection.svg - :width: 200 - :alt: Alternative text - - +++ - - Discover unexpected items in data. - - .. grid-item-card:: - :class-footer: sd-bg-light - - **Simulation** - ^^^ - - .. image:: ../Image/simulation.svg - :width: 200 - :alt: Alternative text - - +++ - - Generate similar data as history data. -``` +# Chronos Installation --- -### **2. Install** + +#### **OS and Python version requirement** + + +```eval_rst +.. note:: + **Supported OS**: + + Chronos is thoroughly tested on Ubuntu (16.04/18.04/20.04), and should works fine on CentOS. If you are a Windows user, the most convenient way to use Chronos on a windows laptop might be using WSL2, you may refer to https://docs.microsoft.com/en-us/windows/wsl/setup/environment or just install a ubuntu virtual machine. +``` +```eval_rst +.. note:: + **Supported Python Version**: + + Chronos only supports Python 3.7.2 ~ latest 3.7.x. We are validating more Python versions. +``` + + + +#### **Install using Conda** + +We recommend using conda to manage the Chronos python environment. For more information about Conda, refer to [here](https://docs.conda.io/en/latest/miniconda.html#). +Select your preferences in the panel below to find the proper install command. Then run the install command as the example shown below. + ```eval_rst .. raw:: html @@ -61,7 +32,7 @@ You can use _Chronos_ to:
    - + @@ -131,97 +102,22 @@ You can use _Chronos_ to:
    - + ```
    -#### **2.1 Pypi** -When you install `bigdl-chronos` from PyPI. We recommend to install with a conda virtual environment. To install Conda, please refer to [here](https://docs.conda.io/en/latest/miniconda.html#). + ```bash +# create a conda environment for chronos conda create -n my_env python=3.7 setuptools=58.0.4 conda activate my_env -# click the installation panel above to find which installation option to use -pip install --pre --upgrade bigdl-chronos[pytorch] # or other options you may want to use + +# select your preference in above panel to find the proper command to replace the below command, e.g. +pip install --pre --upgrade bigdl-chronos[pytorch] + +# init bigdl-nano to enable local accelerations source bigdl-nano-init # accelerate the conda env ``` -#### **2.2 OS and Python version requirement** -```eval_rst -.. note:: - **Supported OS**: - - Chronos is thoroughly tested on Ubuntu (16.04/18.04/20.04), and should works fine on CentOS. If you are a Windows user, the most convenient way to use Chronos on a windows laptop might be using WSL2, you may refer to https://docs.microsoft.com/en-us/windows/wsl/setup/environment or just install a ubuntu virtual machine. -``` -```eval_rst -.. note:: - **Supported Python Version**: - - Chronos only supports Python 3.7.2 ~ latest 3.7.x. We are validating more Python versions. -``` - ---- - - -### **3. Which document to see?** - -```eval_rst -.. grid:: 2 - :gutter: 1 - - .. grid-item-card:: - :class-footer: sd-bg-light - - **Quick Tour** - ^^^ - - You may understand the basic usage of Chronos' components and learn to write the first runnable application in this quick tour page. - - +++ - `Quick Tour <./quick-tour.html>`_ - - .. grid-item-card:: - :class-footer: sd-bg-light - - **User Guides** - ^^^ - - Our user guides provide you with in-depth information, concepts and knowledges about Chronos. - - +++ - - `Data <./data_processing_feature_engineering.html>`_ / - `Forecast <./forecasting.html>`_ / - `Detect <./anomaly_detection.html>`_ / - `Simulate <./simulation.html>`_ - -.. grid:: 2 - :gutter: 1 - - .. grid-item-card:: - :class-footer: sd-bg-light - - **How-to-Guide** / **Example** - ^^^ - - If you are meeting with some specific problems during the usage, how-to guides are good place to be checked. - Examples provides short, high quality use case that users can emulated in their own works. - - +++ - - `How-to-Guide <../Howto/index.html>`_ / `Example <../QuickStart/index.html>`_ - - .. grid-item-card:: - :class-footer: sd-bg-light - - **API Document** - ^^^ - - API Document provides you with a detailed description of the Chronos APIs. - - +++ - - `API Document <../../PythonAPI/Chronos/index.html>`_ - -``` diff --git a/docs/readthedocs/source/doc/Chronos/Overview/quick-tour.rst b/docs/readthedocs/source/doc/Chronos/Overview/quick-tour.rst index 275b4306..58089e6d 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/quick-tour.rst +++ b/docs/readthedocs/source/doc/Chronos/Overview/quick-tour.rst @@ -1,15 +1,11 @@ Chronos Quick Tour -====================== +================================= Welcome to Chronos for building a fast, accurate and scalable time series analysis application🎉! Start with our quick tour to understand some critical concepts and how to use them to tackle your tasks. .. grid:: 1 1 1 1 .. grid-item-card:: :text-align: center - :shadow: none - :class-header: sd-bg-light - :class-footer: sd-bg-light - :class-card: sd-mb-2 **Data processing** ^^^ @@ -22,13 +18,11 @@ Welcome to Chronos for building a fast, accurate and scalable time series analys Get Started -.. grid:: 1 1 3 3 +.. grid:: 1 3 3 3 + :gutter: 2 .. grid-item-card:: :text-align: center - :shadow: none - :class-header: sd-bg-light - :class-footer: sd-bg-light :class-card: sd-mb-2 **Forecasting** @@ -42,11 +36,8 @@ Welcome to Chronos for building a fast, accurate and scalable time series analys Get Started - .. grid-item-card:: + .. grid-item-card:: :text-align: center - :shadow: none - :class-header: sd-bg-light - :class-footer: sd-bg-light :class-card: sd-mb-2 **Anomaly Detection** @@ -60,11 +51,8 @@ Welcome to Chronos for building a fast, accurate and scalable time series analys Get Started - .. grid-item-card:: + .. grid-item-card:: :text-align: center - :shadow: none - :class-header: sd-bg-light - :class-footer: sd-bg-light :class-card: sd-mb-2 **Simulation** @@ -104,7 +92,7 @@ In Chronos, we provide a ``TSDataset`` (and a ``XShardsTSDataset`` to handle lar .. grid:: 2 - :gutter: 1 + :gutter: 2 .. grid-item-card:: @@ -192,7 +180,7 @@ For time series forecasting, we also provide an ``AutoTSEstimator`` for distribu stop_orca_context() .. grid:: 3 - :gutter: 1 + :gutter: 2 .. grid-item-card:: @@ -246,7 +234,7 @@ To import a specific detector, you may use {algorithm name} + "Detector", and ca anomaly_indexes = detector.anomaly_indexes() .. grid:: 3 - :gutter: 1 + :gutter: 2 .. grid-item-card:: @@ -280,7 +268,7 @@ Simulator(experimental) Simulator is still under activate development with unstable API. .. grid:: 2 - :gutter: 1 + :gutter: 2 .. grid-item-card:: diff --git a/docs/readthedocs/source/doc/Chronos/Overview/simulation.md b/docs/readthedocs/source/doc/Chronos/Overview/simulation.md index 274256ab..373a6757 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/simulation.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/simulation.md @@ -1,17 +1,17 @@ -# Generate Synthetic Sequential Data Overview +# Synthetic Data Generation Chronos provides simulators to generate synthetic time series data for users who want to conquer limited data access in a deep learning/machine learning project or only want to generate some synthetic data to play with. ```eval_rst .. note:: - DPGANSimulator is the only simulator chronos provides at the moment, more simulators are on their way. + ``DPGANSimulator`` is the only simulator chronos provides at the moment, more simulators are on their way. ``` ## **1. DPGANSimulator** `DPGANSimulator` adopt DoppelGANger raised in [Using GANs for Sharing Networked Time Series Data: Challenges, Initial Promise, and Open Questions](http://arxiv.org/abs/1909.13403). The method is data-driven unsupervised method based on deep learning model with GAN (Generative Adversarial Networks) structure. The model features a pair of seperate attribute generator and feature generator and their corresponding discriminators `DPGANSimulator` also supports a rich and comprehensive input data (training data) format and outperform other algorithms in many evalution metrics. ```eval_rst -.. note:: +.. note:: We reimplement this model by pytorch(original implementation was based on tf1) for better performance(both speed and memory). ``` diff --git a/docs/readthedocs/source/doc/Chronos/Overview/speed_up.md b/docs/readthedocs/source/doc/Chronos/Overview/speed_up.md index fbc2a981..52f729be 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/speed_up.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/speed_up.md @@ -1,4 +1,4 @@ -# Speed up Chronos built-in models/customized time-series models +# Accelerated Training and Inference Chronos provides transparent acceleration for Chronos built-in models and customized time-series models. In this deep-dive page, we will introduce how to enable/disable them. @@ -16,7 +16,7 @@ Time series model, especially those deep learning models, often suffers slow tra ### **2. Training Acceleration** Training Acceleration is transparent in Chronos's API. Transparentness means that Chronos users will enjoy the acceleration without changing their code(unless some expert users want to set some advanced settings). ```eval_rst -.. note:: +.. note:: **Write your script under** ``if __name__=="__main__":``: Chronos will automatically utilize the computation resources on the hardware. This may include multi-process training on a single node. Use this header will prevent many strange behavior. @@ -65,7 +65,7 @@ We have examples adapted from `pytorch-forecasting`'s examples to show the signi We are working on the acceleration of `AutoModel` and `AutoTSEstimator`. Please unset the environment by: ```bash source bigdl-nano-unset-env -``` +``` ### **3. Inference Acceleration** Inference has become a critical part for time series model's performance. This may be divided to two parts: @@ -77,10 +77,10 @@ Typically, throughput and latency is a trade-off pair. We have three optimizatio - **ONNX Runtime**: Users may export their trained(w/wo auto tuning) model to ONNX file and deploy it on other service. Chronos also provides an internal onnxruntime inference support for those users who pursue low latency and higher throughput during inference on a single node. - **Quantization**: Quantization refers to processes that enable lower precision inference. In Chronos, post-training quantization is supported relied on [Intel® Neural Compressor](https://intel.github.io/neural-compressor/README.html). ```eval_rst -.. note:: +.. note:: **Additional Dependencies**: - You need to install `neural-compressor` to enable quantization related methods. + You need to install ``neural-compressor`` to enable quantization related methods. ``pip install neural-compressor==1.8.1`` ``` diff --git a/docs/readthedocs/source/doc/Chronos/Overview/useful_functionalities.md b/docs/readthedocs/source/doc/Chronos/Overview/useful_functionalities.md index 082ba71b..c45acfc8 100644 --- a/docs/readthedocs/source/doc/Chronos/Overview/useful_functionalities.md +++ b/docs/readthedocs/source/doc/Chronos/Overview/useful_functionalities.md @@ -1,56 +1,7 @@ -# Useful Functionalities Overview +# Distributed Processing -#### **1. AutoML Visualization** -AutoML visualization provides two kinds of visualization. You may use them while fitting on auto models or AutoTS pipeline. -* During the searching process, the visualizations of each trail are shown and updated every 30 seconds. (Monitor view) -* After the searching process, a leaderboard of each trail's configs and metrics is shown. (Leaderboard view) - -**Note**: AutoML visualization is based on tensorboard and tensorboardx. They should be installed properly before the training starts. - -**Monitor view** - -Before training, start the tensorboard server through - -```python -tensorboard --logdir=/ -``` - -`logs_dir` is the log directory you set for your predictor(e.g. `AutoTSEstimator`, `AutoTCN`, etc.). `name ` is the name parameter you set for your predictor. - -The data in SCALARS tag will be updated every 30 seconds for users to see the training progress. - -![](../Image/automl_monitor.png) - -After training, start the tensorboard server through - -```python -tensorboard --logdir=/_leaderboard/ -``` - -where `logs_dir` and `name` are the same as stated in [Monitor view](#monitor_view). - -A dashboard of each trail's configs and metrics is shown in the SCALARS tag. - -![](../Image/automl_scalars.png) - -A leaderboard of each trail's configs and metrics is shown in the HPARAMS tag. - -![](../Image/automl_hparams.png) - -**Use visualization in Jupyter Notebook** - -You can enable a tensorboard view in jupyter notebook by the following code. - -```python -%load_ext tensorboard -# for scalar view -%tensorboard --logdir // -# for leaderboard view -%tensorboard --logdir /_leaderboard/ -``` - -#### **2. Distributed training** +#### **Distributed training** LSTM, TCN and Seq2seq users can easily train their forecasters in a distributed fashion to **handle extra large dataset and utilize a cluster**. The functionality is powered by Project Orca. ```python f = Forecaster(..., distributed=True) @@ -59,10 +10,10 @@ f.predict(...) f.to_local() # collect the forecaster to single node f.predict_with_onnx(...) # onnxruntime only supports single node ``` -#### **3. XShardsTSDataset** +#### **Distributed Data processing: XShardsTSDataset** ```eval_rst .. warning:: - `XShardsTSDataset` is still experimental. + ``XShardsTSDataset`` is still experimental. ``` `TSDataset` is a single thread lib with reasonable speed on large datasets(~10G). When you handle an extra large dataset or limited memory on a single node, `XShardsTSDataset` can be involved to handle the exact same functionality and usage as `TSDataset` in a distributed fashion. diff --git a/docs/readthedocs/source/doc/Chronos/Overview/visualization.md b/docs/readthedocs/source/doc/Chronos/Overview/visualization.md new file mode 100644 index 00000000..146870c9 --- /dev/null +++ b/docs/readthedocs/source/doc/Chronos/Overview/visualization.md @@ -0,0 +1,49 @@ +# AutoML Visualization + +AutoML visualization provides two kinds of visualization. You may use them while fitting on auto models or AutoTS pipeline. +* During the searching process, the visualizations of each trail are shown and updated every 30 seconds. (Monitor view) +* After the searching process, a leaderboard of each trail's configs and metrics is shown. (Leaderboard view) + +**Note**: AutoML visualization is based on tensorboard and tensorboardx. They should be installed properly before the training starts. + +**Monitor view** + +Before training, start the tensorboard server through + +```python +tensorboard --logdir=/ +``` + +`logs_dir` is the log directory you set for your predictor(e.g. `AutoTSEstimator`, `AutoTCN`, etc.). `name ` is the name parameter you set for your predictor. + +The data in SCALARS tag will be updated every 30 seconds for users to see the training progress. + +![](../Image/automl_monitor.png) + +After training, start the tensorboard server through + +```python +tensorboard --logdir=/_leaderboard/ +``` + +where `logs_dir` and `name` are the same as stated in [Monitor view](#monitor_view). + +A dashboard of each trail's configs and metrics is shown in the SCALARS tag. + +![](../Image/automl_scalars.png) + +A leaderboard of each trail's configs and metrics is shown in the HPARAMS tag. + +![](../Image/automl_hparams.png) + +**Use visualization in Jupyter Notebook** + +You can enable a tensorboard view in jupyter notebook by the following code. + +```python +%load_ext tensorboard +# for scalar view +%tensorboard --logdir // +# for leaderboard view +%tensorboard --logdir /_leaderboard/ +``` \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Chronos/QuickStart/chronos-tsdataset-forecaster-quickstart.md b/docs/readthedocs/source/doc/Chronos/QuickStart/chronos-tsdataset-forecaster-quickstart.md index 80c19bc1..cde45dce 100644 --- a/docs/readthedocs/source/doc/Chronos/QuickStart/chronos-tsdataset-forecaster-quickstart.md +++ b/docs/readthedocs/source/doc/Chronos/QuickStart/chronos-tsdataset-forecaster-quickstart.md @@ -8,7 +8,7 @@ **In this guide we will demonstrate how to use _Chronos TSDataset_ and _Chronos Forecaster_ for time seires processing and forecasting in 4 simple steps.** -### **Step 0: Prepare Environment** +### Step 0: Prepare Environment We recommend using [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to prepare the environment. Please refer to the [install guide](../Overview/chronos.html#install) for more details. diff --git a/docs/readthedocs/source/doc/Chronos/index.rst b/docs/readthedocs/source/doc/Chronos/index.rst new file mode 100644 index 00000000..24c9c519 --- /dev/null +++ b/docs/readthedocs/source/doc/Chronos/index.rst @@ -0,0 +1,89 @@ +BigDL-Chronos +======================== + +**BigDL-Chronos** (**Chronos** for short) is an application framework for building a fast, accurate and scalable time series analysis application. + +You can use **Chronos** for: + +.. grid:: 1 3 3 3 + + .. grid-item:: + + .. image:: ./Image/forecasting.svg + :alt: Forcasting example diagram + + **Forecasting:** Predict future using history data. + + .. grid-item:: + + .. image:: ./Image/anomaly_detection.svg + :alt: Anomaly Detection example diagram + + **Anomaly Detection:** Discover unexpected items in data. + + .. grid-item:: + + .. image:: ./Image/simulation.svg + :alt: Simulation example diagram + + **Simulation:** Generate similar data as history data. + +------- + + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + You may understand the basic usage of Chronos' components and learn to write the first runnable application in this quick tour page. + + +++ + :bdg-link:`Chronos in 5 minutes <./Overview/quick-tour.html>` | + :bdg-link:`Installation <./Overview/install.html>` + + .. grid-item-card:: + + **Key Features Guide** + ^^^ + + Our user guides provide you with in-depth information, concepts and knowledges about Chronos. + + +++ + + :bdg-link:`Data <./Overview/data_processing_feature_engineering.html>` | + :bdg-link:`Forecast <./Overview/forecasting.html>` | + :bdg-link:`Detect <./Overview/anomaly_detection.html>` | + :bdg-link:`Simulate <./Overview/simulation.html>` + + .. grid-item-card:: + + **How-to-Guide** / **Tutorials** + ^^^ + + If you are meeting with some specific problems during the usage, how-to guides are good place to be checked. + Examples provides short, high quality use case that users can emulated in their own works. + + +++ + + :bdg-link:`How-to-Guide <./Howto/index.html>` | :bdg-link:`Example <./QuickStart/index.html>` + + .. grid-item-card:: + + **API Document** + ^^^ + + API Document provides you with a detailed description of the Chronos APIs. + + +++ + + :bdg-link:`API Document <../PythonAPI/Chronos/index.html>` + + +.. toctree:: + :hidden: + + BigDL-Chronos Document \ No newline at end of file diff --git a/docs/readthedocs/source/doc/DLlib/Image/tensorboard-histo1.png b/docs/readthedocs/source/doc/DLlib/Image/tensorboard-histo1.png new file mode 100644 index 00000000..68e716d6 Binary files /dev/null and b/docs/readthedocs/source/doc/DLlib/Image/tensorboard-histo1.png differ diff --git a/docs/readthedocs/source/doc/DLlib/Image/tensorboard-histo2.png b/docs/readthedocs/source/doc/DLlib/Image/tensorboard-histo2.png new file mode 100644 index 00000000..bb1db0a7 Binary files /dev/null and b/docs/readthedocs/source/doc/DLlib/Image/tensorboard-histo2.png differ diff --git a/docs/readthedocs/source/doc/DLlib/Image/tensorboard-scalar.png b/docs/readthedocs/source/doc/DLlib/Image/tensorboard-scalar.png new file mode 100644 index 00000000..0f593b71 Binary files /dev/null and b/docs/readthedocs/source/doc/DLlib/Image/tensorboard-scalar.png differ diff --git a/docs/readthedocs/source/doc/DLlib/Overview/dllib.md b/docs/readthedocs/source/doc/DLlib/Overview/dllib.md index 1f818ab6..6ce6269b 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/dllib.md +++ b/docs/readthedocs/source/doc/DLlib/Overview/dllib.md @@ -1,140 +1,81 @@ -# DLlib User Guide +# DLlib in 5 minutes -## 1. Overview +## Overview DLlib is a distributed deep learning library for Apache Spark; with DLlib, users can write their deep learning applications as standard Spark programs (using either Scala or Python APIs). It includes the functionalities of the [original BigDL](https://github.com/intel-analytics/BigDL/tree/branch-0.14) project, and provides following high-level APIs for distributed deep learning on Spark: -* [Keras-like API](keras-api.md) +* [Keras-like API](keras-api.md) * [Spark ML pipeline support](nnframes.md) -## 2. Scala user guide -### 2.1 Install and Run -Please refer [scala guide](../../UserGuide/scala.md) for details. --- -### 2.2 Get started ---- +## Scala Example This section show a single example of how to use dllib to build a deep learning application on Spark, using Keras APIs ---- #### **LeNet Model on MNIST using Keras-Style API** This tutorial is an explanation of what is happening in the [lenet](https://github.com/intel-analytics/BigDL/tree/branch-2.0/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras) example A bigdl-dllib program starts with initialize as follows. ````scala - val conf = Engine.createSparkConf() - .setAppName("Train Lenet on MNIST") - .set("spark.task.maxFailures", "1") - val sc = new SparkContext(conf) - Engine.init +val conf = Engine.createSparkConf() + .setAppName("Train Lenet on MNIST") + .set("spark.task.maxFailures", "1") +val sc = new SparkContext(conf) +Engine.init ```` After the initialization, we need to: 1. Load train and validation data by _**creating the [```DataSet```](https://github.com/intel-analytics/BigDL/blob/branch-2.0/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/feature/dataset/DataSet.scala)**_ (e.g., ````SampleToGreyImg````, ````GreyImgNormalizer```` and ````GreyImgToBatch````): + ````scala + val trainSet = (if (sc.isDefined) { + DataSet.array(load(trainData, trainLabel), sc.get, param.nodeNumber) + } else { + DataSet.array(load(trainData, trainLabel)) + }) -> SampleToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch( + param.batchSize) -````scala - val trainSet = (if (sc.isDefined) { - DataSet.array(load(trainData, trainLabel), sc.get, param.nodeNumber) - } else { - DataSet.array(load(trainData, trainLabel)) - }) -> SampleToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch( - param.batchSize) - - val validationSet = DataSet.array(load(validationData, validationLabel), sc) -> - BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch( - param.batchSize) -```` + val validationSet = DataSet.array(load(validationData, validationLabel), sc) -> + BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch( + param.batchSize) + ```` 2. We then define Lenet model using Keras-style api -````scala - val input = Input(inputShape = Shape(28, 28, 1)) - val reshape = Reshape(Array(1, 28, 28)).inputs(input) - val conv1 = Convolution2D(6, 5, 5, activation = "tanh").setName("conv1_5x5").inputs(reshape) - val pool1 = MaxPooling2D().inputs(conv1) - val conv2 = Convolution2D(12, 5, 5, activation = "tanh").setName("conv2_5x5").inputs(pool1) - val pool2 = MaxPooling2D().inputs(conv2) - val flatten = Flatten().inputs(pool2) - val fc1 = Dense(100, activation = "tanh").setName("fc1").inputs(flatten) - val fc2 = Dense(classNum, activation = "softmax").setName("fc2").inputs(fc1) - Model(input, fc2) - ```` + ````scala + val input = Input(inputShape = Shape(28, 28, 1)) + val reshape = Reshape(Array(1, 28, 28)).inputs(input) + val conv1 = Convolution2D(6, 5, 5, activation = "tanh").setName("conv1_5x5").inputs(reshape) + val pool1 = MaxPooling2D().inputs(conv1) + val conv2 = Convolution2D(12, 5, 5, activation = "tanh").setName("conv2_5x5").inputs(pool1) + val pool2 = MaxPooling2D().inputs(conv2) + val flatten = Flatten().inputs(pool2) + val fc1 = Dense(100, activation = "tanh").setName("fc1").inputs(flatten) + val fc2 = Dense(classNum, activation = "softmax").setName("fc2").inputs(fc1) + Model(input, fc2) + ```` 3. After that, we configure the learning process. Set the ````optimization method```` and the ````Criterion```` (which, given input and target, computes gradient per given loss function): -````scala - model.compile(optimizer = optimMethod, - loss = ClassNLLCriterion[Float](logProbAsInput = false), - metrics = Array(new Top1Accuracy[Float](), new Top5Accuracy[Float](), new Loss[Float])) -```` + ````scala + model.compile(optimizer = optimMethod, + loss = ClassNLLCriterion[Float](logProbAsInput = false), + metrics = Array(new Top1Accuracy[Float](), new Top5Accuracy[Float](), new Loss[Float])) + ```` Finally we _**train the model**_ by calling ````model.fit````: ````scala - model.fit(trainSet, nbEpoch = param.maxEpoch, validationData = validationSet) +model.fit(trainSet, nbEpoch = param.maxEpoch, validationData = validationSet) ```` --- -## 3. Python user guide +## Python Example -### 3.1 Install - -#### 3.1.1 Official Release - -Run below command to install _bigdl-dllib_. - -```bash -conda create -n my_env python=3.7 -conda activate my_env -pip install bigdl-dllib -``` - -#### 3.1.2 Nightly build - -You can install the latest nightly build of bigdl-dllib as follows: -```bash -pip install --pre --upgrade bigdl-dllib -``` - - -### 3.2 Run - -#### **3.2.1 Interactive Shell** - -You may test if the installation is successful using the interactive Python shell as follows: - -* Type `python` in the command line to start a REPL. -* Try to run the example code below to verify the installation: - - ```python - from bigdl.dllib.utils.nncontext import * - - sc = init_nncontext() # Initiation of bigdl-dllib on the underlying cluster. - ``` - -#### **3.2.2 Jupyter Notebook** - -You can start the Jupyter notebook as you normally do using the following command and run bigdl-dllib programs directly in a Jupyter notebook: - -```bash -jupyter notebook --notebook-dir=./ --ip=* --no-browser -``` - -#### **3.2.3 Python Script** - -You can directly write bigdl-dlllib programs in a Python file (e.g. script.py) and run in the command line as a normal Python program: - -```bash -python script.py -``` ---- -### 3.3 Get started - -#### **NN Context** +#### **Initialize NN Context** `NNContext` is the main entry for provisioning the dllib program on the underlying cluster (such as K8s or Hadoop cluster), or just on a single laptop. @@ -158,15 +99,15 @@ This tutorial describes the [Autograd](https://github.com/intel-analytics/BigDL/ The example first do the initializton using `init_nncontext()`: ```python - sc = init_nncontext() +sc = init_nncontext() ``` It then generate the input data X_, Y_ ```python - data_len = 1000 - X_ = np.random.uniform(0, 1, (1000, 2)) - Y_ = ((2 * X_).sum(1) + 0.4).reshape([data_len, 1]) +data_len = 1000 +X_ = np.random.uniform(0, 1, (1000, 2)) +Y_ = ((2 * X_).sum(1) + 0.4).reshape([data_len, 1]) ``` It then define the custom loss @@ -179,20 +120,20 @@ def mean_absolute_error(y_true, y_pred): After that, the example creates the model as follows and set the criterion as the custom loss: ```python - a = Input(shape=(2,)) - b = Dense(1)(a) - c = Lambda(function=add_one_func)(b) - model = Model(input=a, output=c) +a = Input(shape=(2,)) +b = Dense(1)(a) +c = Lambda(function=add_one_func)(b) +model = Model(input=a, output=c) - model.compile(optimizer=SGD(learningrate=1e-2), - loss=mean_absolute_error) +model.compile(optimizer=SGD(learningrate=1e-2), + loss=mean_absolute_error) ``` Finally the example trains the model by calling `model.fit`: ```python - model.fit(x=X_, - y=Y_, - batch_size=32, - nb_epoch=int(options.nb_epoch), - distributed=False) +model.fit(x=X_, + y=Y_, + batch_size=32, + nb_epoch=int(options.nb_epoch), + distributed=False) ``` \ No newline at end of file diff --git a/docs/readthedocs/source/doc/DLlib/Overview/index.rst b/docs/readthedocs/source/doc/DLlib/Overview/index.rst new file mode 100644 index 00000000..e2fd61e5 --- /dev/null +++ b/docs/readthedocs/source/doc/DLlib/Overview/index.rst @@ -0,0 +1,6 @@ +DLLib Key Features +================================ + +* `Keras-like API `_ +* `Spark ML Pipeline Support `_ +* `Visualization `_ \ No newline at end of file diff --git a/docs/readthedocs/source/doc/DLlib/Overview/install.md b/docs/readthedocs/source/doc/DLlib/Overview/install.md new file mode 100644 index 00000000..a64f3892 --- /dev/null +++ b/docs/readthedocs/source/doc/DLlib/Overview/install.md @@ -0,0 +1,41 @@ +# Installation + + +## Scala + +Refer to [BigDl Install guide for Scala](../../UserGuide/scala.md). + + +## Python + + +### Install a Stable Release + +Run below command to install _bigdl-dllib_. + +```bash +conda create -n my_env python=3.7 +conda activate my_env +pip install bigdl-dllib +``` + +### Install Nightly build version + +You can install the latest nightly build of bigdl-dllib as follows: +```bash +pip install --pre --upgrade bigdl-dllib +``` + +### Verify your install + +You may verify if the installation is successful using the interactive Python shell as follows: + +* Type `python` in the command line to start a REPL. +* Try to run the example code below to verify the installation: + + ```python + from bigdl.dllib.utils.nncontext import * + + sc = init_nncontext() # Initiation of bigdl-dllib on the underlying cluster. + ``` + diff --git a/docs/readthedocs/source/doc/DLlib/Overview/keras-api.md b/docs/readthedocs/source/doc/DLlib/Overview/keras-api.md index 3e4ea656..71504d0b 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/keras-api.md +++ b/docs/readthedocs/source/doc/DLlib/Overview/keras-api.md @@ -1,4 +1,4 @@ -# Keras-Like API +# Keras-Like API ## 1. Introduction [DLlib](dllib.md) provides __Keras-like API__ based on [__Keras 1.2.2__](https://faroit.github.io/keras-docs/1.2.2/) for distributed deep learning on Apache Spark. Users can easily use the Keras-like API to create a neural network model, and train, evaluate or tune it in a distributed fashion on Spark. @@ -144,2699 +144,10 @@ val output = merge(inputs = List(dense1, dense2), mode = "sum") val model = Model[Float](Array(input1, input2), output) ``` ---- -## 7. Core Layers -This section describes all the available layers in the Keras-like API. - -To set the name of a specific layer, you call the method `setName(name)` of that layer. - -### 7.1 Masking -Use a mask value to skip timesteps for a sequence. - -**Scala:** -```scala -Masking(maskValue = 0.0, inputShape = null) -``` -**Python:** -```python -Masking(mask_value=0.0, input_shape=None, name=None) -``` - -**Parameters:** - -* `maskValue`: Mask value. For each timestep in the input (the second dimension), if all the values in the input at that timestep are equal to 'maskValue', then the timestep will be masked (skipped) in all downstream layers. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Masking -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Masking[Float](inputShape = Shape(3))) -val input = Tensor[Float](2, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -1.4539868 1.5623108 -1.4101523 -0.77073747 -0.18994702 2.2574463 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -1.4539868 1.5623108 -1.4101523 -0.77073747 -0.18994702 2.2574463 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` - -**Python example:** -```python -import numpy as np -from bigdl.nn.keras.topology import Sequential -from bigdl.nn.keras.layer import Masking - -model = Sequential() -model.add(Masking(input_shape=(3, ))) -input = np.random.random([2, 3]) -output = model.forward(input) -``` -Input is: -```python -[[0.31542103 0.20640659 0.22282763] - [0.99352167 0.90135718 0.24504717]] -``` -Output is -```python -[[0.31542102 0.2064066 0.22282763] - [0.9935217 0.9013572 0.24504717]] -``` - ---- -### 7.2 SparseDense -SparseDense is the sparse version of layer Dense. SparseDense has two different from Dense: -firstly, SparseDense's input Tensor is a SparseTensor. Secondly, SparseDense doesn't backward -gradient to next layer in the backpropagation by default, as the gradInput of SparseDense is -useless and very big in most cases. - -But, considering model like Wide&Deep, we provide backwardStart and backwardLength to backward -part of the gradient to next layer. - -The most common input is 2D. - -**Scala:** -```scala -SparseDense(outputDim, init = "glorot_uniform", activation = null, wRegularizer = null, bRegularizer = null, backwardStart = -1, backwardLength = -1, initWeight = null, initBias = null, initGradWeight = null, initGradBias = null, bias = true, inputShape = null) -``` -**Python:** -```python -SparseDense(output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, backward_start=-1, backward_length=-1, init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, bias=True, input_shape=None, name=None) -``` - -**Parameters:** - -* `outputDim`: The size of the output dimension. -* `init`: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. -* `activation`: String representation of the activation function to use. Default is null. -* `wRegularizer`: An instance of [Regularizer], applied to the input weights matrices. Default is null. -* `bRegularizer`: An instance of [Regularizer], applied to the bias. Default is null. -* `bias`: Whether to include a bias (i.e. make the layer affine rather than linear). Default is true. -* `backwardStart`: Backward start index, counting from 1. -* `backwardLength`: Backward length. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a `Shape` object. For Python API, it should be a shape tuple. Batch dimension should be excluded. -* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.SparseDense -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val layer = SparseDense[Float](outputDim = 5, inputShape = Shape(2, 4)) -layer.build(Shape(-1, 2, 4)) -val input = Tensor[Float](Array(2, 4)).rand() -input.setValue(1, 1, 1f) -input.setValue(2, 3, 3f) -val sparseInput = Tensor.sparse(input) -val output = layer.forward(sparseInput) -``` -Input is: -```scala -input: -(0, 0) : 1.0 -(0, 1) : 0.2992794 -(0, 2) : 0.11227019 -(0, 3) : 0.722947 -(1, 0) : 0.6147614 -(1, 1) : 0.4288646 -(1, 2) : 3.0 -(1, 3) : 0.7749917 -[com.intel.analytics.bigdl.tensor.SparseTensor of size 2x4] -``` -Output is: -```scala -output: -0.053516 0.33429605 0.22587383 -0.8998945 0.24308181 -0.76745665 -1.614114 0.5381658 -2.2226436 -0.15573677 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x5] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential -from bigdl.dllib.utils.common import JTensor - -model = Sequential() -model.add(SparseDense(output_dim=2, input_shape=(3, 4))) -input = JTensor.sparse( - a_ndarray=np.array([1, 3, 2, 4]), - i_ndarray = np.array([[0, 0, 1, 2], - [0, 3, 2, 1]]), - shape = np.array([3, 4]) -) -output = model.forward(input) -``` -Input is: -```python -JTensor: storage: [1. 3. 2. 4.], shape: [3 4] ,indices [[0 0 1 2] - [0 3 2 1]], float -``` -Output is -```python -[[ 1.57136 2.29596 ] - [ 0.5791738 -1.6598101 ] - [ 2.331141 -0.84687066]] - ``` - -### 7.3 SoftShrink -Applies the soft shrinkage function element-wise to the input. - -When you use this layer as the first layer of a model, you need to provide -the argument inputShape (a Single Shape, does not include the batch dimension). - -Remark: This layer is from Torch and wrapped in Keras style. - - -**Scala:** -```scala -SoftShrink(value = 0.5, inputShape = null) -``` -**Python:** -```python -SoftShrink(value = 0.5, input_shape=None, name=None) -``` - -**Parameters:** - -* `value`: value The threshold value. Default is 0.5. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a `Shape` object. For Python API, it should be a shape tuple. Batch dimension should be excluded. -* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.SoftShrink -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(SoftShrink[Float](0.6, inputShape = Shape(2, 3, 4))) -val input = Tensor[Float](2, 2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = --0.36938807 0.023556225 -1.1655436 -0.34449077 -0.9444338 -0.086538695 -1.0425501 1.364976 --1.2563878 -0.1842559 0.43428117 1.0756494 - -(1,2,.,.) = --0.19888283 1.251872 0.114836805 -0.6208773 -0.0051822234 -0.8998633 0.06937465 -0.3929931 --0.1058129 0.6945743 -0.40083578 -0.6252444 - -(2,1,.,.) = --0.9899709 -0.77926594 -0.15497442 -0.15031165 --0.6028622 0.86623466 -2.1543107 0.41970536 --0.8215522 0.3014275 -0.32184362 0.14445356 - -(2,2,.,.) = -0.74701905 0.10044397 -0.40519297 0.03822808 -0.30726334 0.27862388 1.731753 0.032177072 --1.3476961 -0.2294767 0.99794704 0.7398458 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.0 0.0 -0.56554353 0.0 -0.34443378 0.0 -0.44255006 0.764976 --0.6563878 0.0 0.0 0.47564936 - -(1,2,.,.) = -0.0 0.6518719 0.0 -0.020877302 -0.0 -0.29986328 0.0 0.0 -0.0 0.09457427 0.0 -0.025244355 - -(2,1,.,.) = --0.3899709 -0.17926592 0.0 0.0 --0.0028621554 0.26623464 -1.5543107 0.0 --0.2215522 0.0 0.0 0.0 - -(2,2,.,.) = -0.14701903 0.0 0.0 0.0 -0.0 0.0 1.131753 0.0 --0.74769604 0.0 0.397947 0.13984579 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(SoftShrink(0.6, input_shape=(2, 3, 4))) -input = np.random.random([2, 2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -array([[[[ 0.43421006, 0.28394451, 0.15221226, 0.47268966], - [ 0.22426224, 0.24855662, 0.790498 , 0.67767582], - [ 0.14879562, 0.56077882, 0.61470262, 0.94875862]], - - [[ 0.72404932, 0.89780875, 0.08456734, 0.01303937], - [ 0.25023568, 0.45392504, 0.587254 , 0.51164461], - [ 0.12277567, 0.05571182, 0.17076456, 0.71660884]]], - - - [[[ 0.06369975, 0.85395557, 0.35752425, 0.606633 ], - [ 0.67640252, 0.86861737, 0.18040722, 0.55467108], - [ 0.24102058, 0.37580645, 0.81601612, 0.56513788]], - - [[ 0.8461435 , 0.65668365, 0.17969807, 0.51602926], - [ 0.86191073, 0.34245714, 0.62795207, 0.36706125], - [ 0.80344028, 0.81056003, 0.80959083, 0.15366483]]]]) -``` -Output is -```python -array([[[[ 0. , 0. , 0. , 0. ], - [ 0. , 0. , 0.19049799, 0.07767582], - [ 0. , 0. , 0.01470262, 0.34875858]], - - [[ 0.12404931, 0.29780871, 0. , 0. ], - [ 0. , 0. , 0. , 0. ], - [ 0. , 0. , 0. , 0.1166088 ]]], - - - [[[ 0. , 0.25395554, 0. , 0.00663298], - [ 0.07640249, 0.26861733, 0. , 0. ], - [ 0. , 0. , 0.21601611, 0. ]], - - [[ 0.24614346, 0.05668366, 0. , 0. ], - [ 0.26191074, 0. , 0.02795208, 0. ], - [ 0.20344025, 0.21056002, 0.20959079, 0. ]]]], dtype=float32) - - ``` - ---- -### 7.4 Reshape -Reshapes an output to a certain shape. - -Supports shape inference by allowing one -1 in the target shape. For example, if input shape is (2, 3, 4), target shape is (3, -1), then output shape will be (3, 8). - -**Scala:** -```scala -Reshape(targetShape, inputShape = null) -``` -**Python:** -```python -Reshape(target_shape, input_shape=None, name=None) -``` - -**Parameters:** - -* `targetShape`: The target shape that you desire to have. Batch dimension should be excluded. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Reshape -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Reshape(Array(3, 8), inputShape = Shape(2, 3, 4))) -val input = Tensor[Float](2, 2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = --1.7092276 -1.3941092 -0.6348466 0.71309644 -0.3605411 0.025597548 0.4287048 -0.548675 -0.4623341 -2.3912702 0.22030865 -0.058272455 - -(1,2,.,.) = --1.5049093 -1.8828062 0.8230564 -0.020209199 --0.3415721 1.1219939 1.1089007 -0.74697906 --1.503861 -1.616539 0.048006497 1.1613717 - -(2,1,.,.) = -0.21216023 1.0107462 0.8586909 -0.05644316 --0.31436008 1.6892323 -0.9961186 -0.08169463 -0.3559391 0.010261055 -0.70408463 -1.2480727 - -(2,2,.,.) = -1.7663039 0.07122444 0.073556066 -0.7847014 -0.17604464 -0.99110585 -1.0302067 -0.39024687 --0.0260166 -0.43142694 0.28443158 0.72679126 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = --1.7092276 -1.3941092 -0.6348466 0.71309644 0.3605411 0.025597548 0.4287048 -0.548675 -0.4623341 -2.3912702 0.22030865 -0.058272455 -1.5049093 -1.8828062 0.8230564 -0.020209199 --0.3415721 1.1219939 1.1089007 -0.74697906 -1.503861 -1.616539 0.048006497 1.1613717 - -(2,.,.) = -0.21216023 1.0107462 0.8586909 -0.05644316 -0.31436008 1.6892323 -0.9961186 -0.08169463 -0.3559391 0.010261055 -0.70408463 -1.2480727 1.7663039 0.07122444 0.073556066 -0.7847014 -0.17604464 -0.99110585 -1.0302067 -0.39024687 -0.0260166 -0.43142694 0.28443158 0.72679126 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x8] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - - -model = Sequential() -model.add(Reshape(target_shape=(3, 8), input_shape=(2, 3, 4))) -input = np.random.random([2, 2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -[[[[0.39260304 0.10383185 0.87490319 0.89167328] - [0.61649117 0.43285247 0.86851582 0.97743004] - [0.90018969 0.04303951 0.74263493 0.14208656]] - [[0.66193405 0.93432157 0.76160537 0.70437459] - [0.99953431 0.23016734 0.42293405 0.66078049] - [0.03357645 0.9695145 0.30111138 0.67109948]]] - - [[[0.39640201 0.92930203 0.86027666 0.13958544] - [0.34584767 0.14743425 0.93804016 0.38053062] - [0.55068792 0.77375329 0.84161166 0.48131356]] - [[0.90116368 0.53253689 0.03332962 0.58278686] - [0.34935685 0.32599554 0.97641892 0.57696434] - [0.53974677 0.90682861 0.20027319 0.05962118]]]] -``` -Output is -```python -[[[0.39260304 0.10383185 0.8749032 0.89167327 0.6164912 0.43285248 0.86851585 0.97743005] - [0.9001897 0.04303951 0.74263495 0.14208655 0.661934 0.9343216 0.7616054 0.7043746 ] - [0.9995343 0.23016734 0.42293406 0.6607805 0.03357645 0.9695145 0.30111137 0.6710995 ]] - - [[0.396402 0.92930204 0.86027664 0.13958544 0.34584767 0.14743425 0.93804014 0.38053063] - [0.5506879 0.7737533 0.8416117 0.48131356 0.9011637 0.53253686 0.03332962 0.58278686] - [0.34935686 0.32599553 0.9764189 0.5769643 0.53974676 0.9068286 0.20027319 0.05962119]]] -``` - ---- -### 7.5 Merge -Used to merge a list of inputs into a single output, following some merge mode. - -Merge must have at least two input layers. - -**Scala:** -```scala -Merge(layers = null, mode = "sum", concatAxis = -1, inputShape = null) -``` -**Python:** -```python -Merge(layers=None, mode="sum", concat_axis=-1, input_shape=None, name=None) -``` - -**Parameters:** - -* `layers`: A list of layer instances. Must be more than one layer. -* `mode`: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'. Default is 'sum'. -* `concatAxis`: Integer, axis to use when concatenating layers. Only specify this when merge mode is 'concat'. Default is -1, meaning the last axis of the input. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object. For Python API, it should be a list of shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.InputLayer -import com.intel.analytics.bigdl.dllib.keras.layers.Merge -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.utils.{Shape, T} -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -val l1 = InputLayer[Float](inputShape = Shape(2, 3)) -val l2 = InputLayer[Float](inputShape = Shape(2, 3)) -val layer = Merge[Float](layers = List(l1, l2), mode = "sum") -model.add(layer) -val input1 = Tensor[Float](2, 2, 3).rand(0, 1) -val input2 = Tensor[Float](2, 2, 3).rand(0, 1) -val input = T(1 -> input1, 2 -> input2) -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.utils.Table = - { - 2: (1,.,.) = - 0.87815475 0.15025006 0.34412447 - 0.07909282 0.008027249 0.111715704 - - (2,.,.) = - 0.52245367 0.2547527 0.35857987 - 0.7718501 0.26783863 0.8642062 - - [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] - 1: (1,.,.) = - 0.5377018 0.28364193 0.3424284 - 0.0075349305 0.9018168 0.9435114 - - (2,.,.) = - 0.09112563 0.88585275 0.3100201 - 0.7910178 0.57497376 0.39764535 - - [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] - } -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -1.4158566 0.433892 0.6865529 -0.08662775 0.90984404 1.0552272 - -(2,.,.) = -0.6135793 1.1406054 0.66859996 -1.5628679 0.8428124 1.2618515 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -l1 = InputLayer(input_shape=(3, 4)) -l2 = InputLayer(input_shape=(3, 4)) -model.add(Merge(layers=[l1, l2], mode='sum')) -input = [np.random.random([2, 3, 4]), np.random.random([2, 3, 4])] -output = model.forward(input) -``` -Input is: -```python -[[[[0.28764351, 0.0236015 , 0.78927442, 0.52646492], - [0.63922826, 0.45101604, 0.4555552 , 0.70105653], - [0.75790798, 0.78551523, 0.00686686, 0.61290369]], - - [[0.00430865, 0.3303661 , 0.59915782, 0.90362298], - [0.26230717, 0.99383052, 0.50630521, 0.99119486], - [0.56138318, 0.68165639, 0.10644523, 0.51860127]]], - - [[[0.84365767, 0.8854741 , 0.84183673, 0.96322321], - [0.49354248, 0.97936826, 0.2266097 , 0.88083622], - [0.11011776, 0.65762034, 0.17446099, 0.76658969]], - - [[0.58266689, 0.86322199, 0.87122999, 0.19031255], - [0.42275118, 0.76379413, 0.21355413, 0.81132937], - [0.97294728, 0.68601731, 0.39871792, 0.63172344]]]] -``` -Output is -```python -[[[1.1313012 0.90907556 1.6311111 1.4896882 ] - [1.1327708 1.4303843 0.6821649 1.5818927 ] - [0.8680257 1.4431355 0.18132785 1.3794935 ]] - - [[0.5869755 1.1935881 1.4703878 1.0939355 ] - [0.68505836 1.7576246 0.71985936 1.8025242 ] - [1.5343305 1.3676738 0.50516313 1.1503248 ]]] -``` - ---- -### 7.6 MaxoutDense -A dense maxout layer that takes the element-wise maximum of linear layers. - -This allows the layer to learn a convex, piecewise linear activation function over the inputs. - -The input of this layer should be 2D. - -**Scala:** -```scala -MaxoutDense(outputDim, nbFeature = 4, wRegularizer = null, bRegularizer = null, bias = true, inputShape = null) -``` -**Python:** -```python -MaxoutDense(output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, name=None) -``` - -**Parameters:** - -* `outputDim`: The size of output dimension. -* `nbFeature`: Number of Dense layers to use internally. Integer. Default is 4. -* `wRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), (eg. L1 or L2 regularization), applied to the input weights matrices. Default is null. -* `bRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the bias. Default is null. -* `bias`: Whether to include a bias (i.e. make the layer affine rather than linear). Default is true. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.MaxoutDense -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(MaxoutDense(2, inputShape = Shape(3))) -val input = Tensor[Float](2, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = --1.3550005 -1.1668127 -1.2882779 -0.83600295 -1.94683 1.323666 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -0.71675766 1.2987505 -0.9871184 0.6634239 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(MaxoutDense(2, input_shape=(3, ))) -input = np.random.random([2, 3]) -output = model.forward(input) -``` -Input is: -```python -[[0.15996114 0.8391686 0.81922903] - [0.52929427 0.35061754 0.88167693]] -``` -Output is -```python -[[0.4479192 0.4842512] - [0.16833156 0.521764 ]] -``` - ---- -### 7.7 Squeeze -Delete the singleton dimension(s). The batch dimension needs to be unchanged. - -For example, if input has size (2, 1, 3, 4, 1): - -Squeeze(1) will give output size (2, 3, 4, 1), - -Squeeze() will give output size (2, 3, 4) - -**Scala:** -```scala -Squeeze(dims = null, inputShape = null) -``` -**Python:** -```python -Squeeze(dim=None, input_shape=None, name=None) -``` - -**Parameters:** - -* `dims`: The dimension(s) to squeeze. 0-based index. Cannot squeeze the batch dimension. The selected dimensions must be singleton, i.e. having size 1. Default is null, and in this case all the non-batch singleton dimensions will be deleted. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Squeeze -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Squeeze[Float](1, inputShape = Shape(1, 1, 32))) -val input = Tensor[Float](1, 1, 1, 32).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = -0.5521966 -1.2199087 0.365958 1.3845297 0.115254946 -0.20352958 2.4912808 0.987046 -0.2115477 3.0530396 -1.0043625 1.4688021 -1.2412603 -0.25383064 0.49164283 -0.40329486 0.26323202 0.7979045 0.025444122 0.47221214 1.3995043 0.48498031 -0.86961967 -0.058370713 -0.85965866 -1.2727696 0.45570874 0.73393697 0.2567143 1.4261572 -0.37773672 -0.7339463 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x1x1x32] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -0.5521966 -1.2199087 0.365958 1.3845297 0.115254946 -0.20352958 2.4912808 0.987046 -0.2115477 3.0530396 -1.0043625 1.4688021 -1.2412603 -0.25383064 0.49164283 -0.40329486 0.26323202 0.7979045 0.025444122 0.47221214 1.3995043 0.48498031 -0.86961967 -0.058370713 -0.85965866 -1.2727696 0.45570874 0.73393697 0.2567143 1.4261572 -0.37773672 -0.7339463 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x1x32] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Squeeze(1, input_shape=(1, 1, 32))) -input = np.random.random([1, 1, 1, 32]) -output = model.forward(input) -``` -Input is: -```python -[[[[0.20585343, 0.47011701, 0.14553177, 0.93915599, 0.57234281, - 0.91631229, 0.32244256, 0.94243351, 0.86595631, 0.73916763, - 0.35898731, 0.65208275, 0.07935983, 0.89313423, 0.68601269, - 0.48919672, 0.28406399, 0.20962799, 0.88071757, 0.45501821, - 0.60931183, 0.46709718, 0.14218838, 0.42517758, 0.9149958 , - 0.0843243 , 0.27302307, 0.75281922, 0.3688931 , 0.86913729, - 0.89774196, 0.77838838]]]] -``` -Output is -```python -[[[0.20585343, 0.470117 , 0.14553176, 0.939156 , 0.5723428 , - 0.9163123 , 0.32244256, 0.94243354, 0.8659563 , 0.73916763, - 0.3589873 , 0.65208274, 0.07935983, 0.89313424, 0.6860127 , - 0.48919672, 0.284064 , 0.20962799, 0.8807176 , 0.45501822, - 0.6093118 , 0.46709716, 0.14218839, 0.42517757, 0.9149958 , - 0.0843243 , 0.27302307, 0.75281924, 0.36889312, 0.8691373 , - 0.897742 , 0.7783884 ]]] -``` - ---- -### 7.8 BinaryThreshold -Threshold the input. - -If an input element is smaller than the threshold value, it will be replaced by 0; otherwise, it will be replaced by 1. - -**Scala:** -```scala -BinaryThreshold(value = 1e-6, inputShape = null) -``` -**Python:** -```python -BinaryThreshold(value=1e-6, input_shape=None, name=None) -``` - -**Parameters:** - -* `value`: The threshold value to compare with. Default is 1e-6. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.BinaryThreshold -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(BinaryThreshold[Float](inputShape = Shape(2, 3, 4))) -val input = Tensor[Float](2, 2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = --1.1907398 -0.18995096 -2.0344417 -1.3789974 --1.8801064 -0.74757665 -0.4339697 0.0058485097 -0.7012256 -0.6363152 2.0156987 -0.5512639 - -(1,2,.,.) = --0.5251603 0.082127444 0.29550993 1.6357868 --1.3828015 -0.11842779 0.3316966 -0.14360528 -0.21216457 -0.117370956 -0.12934707 -0.35854268 - -(2,1,.,.) = --0.9071151 -2.8566089 -0.4796377 -0.915065 --0.8439908 -0.25404388 -0.39926198 -0.15191565 --1.0496653 -0.403675 -1.3591816 0.5311797 - -(2,2,.,.) = -0.53509855 -0.08892822 1.2196561 -0.62759316 --0.47476718 -0.43337926 -0.10406987 1.4035174 --1.7120812 1.1328355 0.9219375 1.3813454 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.0 0.0 0.0 0.0 -0.0 0.0 0.0 1.0 -1.0 0.0 1.0 0.0 - -(1,2,.,.) = -0.0 1.0 1.0 1.0 -0.0 0.0 1.0 0.0 -1.0 0.0 0.0 0.0 - -(2,1,.,.) = -0.0 0.0 0.0 0.0 -0.0 0.0 0.0 0.0 -0.0 0.0 0.0 1.0 - -(2,2,.,.) = -1.0 0.0 1.0 0.0 -0.0 0.0 0.0 1.0 -0.0 1.0 1.0 1.0 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(BinaryThreshold(input_shape=(2, 3, 4))) -input = np.random.random([2, 2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -array([[[[0.30421481, 0.47800487, 0.54249411, 0.90109767], - [0.72650405, 0.53096719, 0.66346109, 0.0589329 ], - [0.12994731, 0.92181174, 0.43129874, 0.97306968]], - - [[0.3031087 , 0.20339982, 0.69034712, 0.40191 ], - [0.57517034, 0.30159448, 0.4801747 , 0.75175084], - [0.8599362 , 0.93523811, 0.34768628, 0.10840162]]], - - - [[[0.46102959, 0.33029002, 0.69340103, 0.32885719], - [0.84405147, 0.03421879, 0.68242578, 0.03560338], - [0.12244515, 0.3610654 , 0.01312785, 0.84485178]], - - [[0.73472287, 0.75707757, 0.77070527, 0.40863145], - [0.01137898, 0.82896826, 0.1498069 , 0.22309423], - [0.92737483, 0.36217222, 0.06679799, 0.33304362]]]]) -``` -Output is -```python -array([[[[1., 1., 1., 1.], - [1., 1., 1., 1.], - [1., 1., 1., 1.]], - - [[1., 1., 1., 1.], - [1., 1., 1., 1.], - [1., 1., 1., 1.]]], - - - [[[1., 1., 1., 1.], - [1., 1., 1., 1.], - [1., 1., 1., 1.]], - - [[1., 1., 1., 1.], - [1., 1., 1., 1.], - [1., 1., 1., 1.]]]], dtype=float32) -``` - ---- -### 7.9 Sqrt -Applies an element-wise square root operation to the input. - -**Scala:** -```scala -Sqrt(inputShape = null) -``` -**Python:** -```python -Sqrt(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Sqrt -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Sqrt[Float](inputShape = Shape(3))) -val input = Tensor[Float](2, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -0.6950394 0.5234307 1.7375475 -0.25833175 0.02685826 -0.6046901 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -0.8336902 0.7234851 1.3181607 -0.50826347 0.16388491 NaN -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Sqrt(input_shape=(3, ))) -input = np.random.random([2, 3]) -output = model.forward(input) -``` -Input is: -```python -[[0.2484558 , 0.65280218, 0.35286984], - [0.19616094, 0.30966802, 0.82148169]] -``` -Output is -```python -[[0.4984534 , 0.80796176, 0.5940285 ], - [0.4429006 , 0.55647826, 0.9063563 ]] -``` - ---- -### 7.10 Mul -Multiply a single scalar factor to the incoming data - -**Scala:** -```scala -Mul(inputShape = null) -``` -**Python:** -```python -Mul(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. -* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. - - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Mul -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Mul[Float](inputShape = Shape(3, 4))) -val input = Tensor[Float](2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = --1.2316265 -2.008802 -1.3908259 -0.61135375 --0.48992255 0.1786112 0.18872596 0.49621895 --0.6931602 -0.919745 -0.09019699 -0.41218707 - -(2,.,.) = --0.3135355 -0.4385771 -0.3317269 1.0412029 --0.8859662 0.17758773 -0.73779273 -0.4445366 -0.3921595 1.6923207 0.014470488 0.4044164 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = --0.59036994 -0.9629025 -0.6666808 -0.29304734 --0.2348403 0.0856158 0.09046422 0.23785843 --0.33226058 -0.44087213 -0.043235175 -0.19757845 - -(2,.,.) = --0.15029064 -0.21022828 -0.15901053 0.49909195 --0.42468053 0.0851252 -0.3536548 -0.21308492 -0.18797839 0.81119984 0.006936308 0.19385365 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Mul(input_shape=(3, 4))) -input = np.random.random([2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -array([[[ 0.22607292, 0.59806062, 0.19428923, 0.22928606], - [ 0.13804536, 0.1615547 , 0.52824658, 0.52794904], - [ 0.4049169 , 0.94109084, 0.58158453, 0.78368633]], - - [[ 0.86233305, 0.47995805, 0.80430949, 0.9931171 ], - [ 0.35179631, 0.33615276, 0.87756877, 0.73560288], - [ 0.29775703, 0.11404466, 0.77695536, 0.97580018]]]) -``` -Output is -```python -array([[[-0.22486402, -0.59486258, -0.1932503 , -0.22805998], - [-0.13730718, -0.1606908 , -0.52542186, -0.52512592], - [-0.40275168, -0.93605846, -0.57847458, -0.77949566]], - - [[-0.85772187, -0.47739154, -0.80000854, -0.9878065 ], - [-0.34991512, -0.33435524, -0.87287611, -0.73166931], - [-0.29616481, -0.11343482, -0.77280068, -0.97058219]]], dtype=float32) -``` - ---- -### 7.11 MulConstant -Multiply the input by a (non-learnable) scalar constant. - -**Scala:** -```scala -MulConstant(constant, inputShape = null) -``` -**Python:** -```python -MulConstant(constant, input_shape=None, name=None) -``` - -**Parameters:** - -* `constant`: The scalar constant to be multiplied. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.MulConstant -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(MulConstant[Float](2.2, inputShape = Shape(3, 4))) -val input = Tensor[Float](2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = --0.16873977 1.0812985 1.0942211 -0.67091423 -1.0086882 0.5915831 0.26184535 -1.361431 -1.5616825 -0.037591368 1.2794676 1.0692137 - -(2,.,.) = -0.29868057 -0.23266982 -0.7679556 -2.209848 --0.13954644 -0.1368473 -0.54510623 1.8397199 --0.58691734 -0.56410027 -1.5567777 0.050648995 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = --0.3712275 2.3788567 2.4072864 -1.4760114 -2.219114 1.3014828 0.57605976 -2.9951482 -3.4357016 -0.08270101 2.8148286 2.3522704 - -(2,.,.) = -0.6570973 -0.5118736 -1.6895024 -4.8616657 --0.3070022 -0.30106407 -1.1992338 4.047384 --1.2912182 -1.2410206 -3.424911 0.11142779 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import * -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(MulConstant(2.2, input_shape=(3, 4))) -input = np.random.random([2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -[[[0.39874191, 0.66634984, 0.23907766, 0.31587494], - [0.78842014, 0.93057835, 0.80739529, 0.71541279], - [0.2231424 , 0.3372844 , 0.94678072, 0.52928034]], - - [[0.60142458, 0.41221671, 0.00890549, 0.32069845], - [0.51122554, 0.76280426, 0.87579418, 0.17182832], - [0.54133184, 0.19814384, 0.92529327, 0.5616615 ]]] -``` -Output is -```python -[[[0.8772322 , 1.4659697 , 0.5259709 , 0.6949249 ], - [1.7345244 , 2.0472724 , 1.7762697 , 1.5739082 ], - [0.4909133 , 0.7420257 , 2.0829177 , 1.1644168 ]], - - [[1.3231341 , 0.9068768 , 0.01959208, 0.7055366 ], - [1.1246961 , 1.6781695 , 1.9267472 , 0.37802234], - [1.19093 , 0.43591645, 2.0356452 , 1.2356553 ]]] -``` - ---- -### 7.12 Scale -Scale is the combination of CMul and CAdd. - -Computes the element-wise product of the input and weight, with the shape of the weight "expand" to match the shape of the input. - -Similarly, perform an expanded bias and perform an element-wise add. - -**Scala:** -```scala -Scale(size, inputShape = null) -``` -**Python:** -```python -Scale(size, input_shape=None, name=None) -``` - -**Parameters:** - -* `size`: Size of the weight and bias. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Scale -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -var array = Array(1, 2) -model.add(Scale[Float](array, inputShape = Shape(3))) -val input = Tensor[Float](2, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = --0.006399727 -0.06412822 -0.2334789 -0.31029955 1.6557469 1.9614618 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -0.09936619 0.57585865 0.20324506 -0.38537437 -0.8598822 -1.0186496 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import Scale -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Scale((2, 1), input_shape=(3, ))) -input = np.random.random([2, 3]) -output = model.forward(input) -``` -Input is: -```python -[[0.7242994 , 0.77888884, 0.71470432], - [0.03058471, 0.00602764, 0.57513629]] -``` -Output is -```python -[[1.0946966 , 1.1255064 , 1.0892813 ], - [0.58151895, 0.5909191 , 0.37307182]] -``` - ---- -### 7.13 Log -Applies a log transformation to the input. - -**Scala:** -```scala -Log(inputShape = null) -``` -**Python:** -```python -Log(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Log -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Log[Float](inputShape = Shape(2, 4, 4))) -val input = Tensor[Float](1, 2, 4, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = -0.38405678 -0.5502389 -0.383079 -0.988537 --0.6294056 -0.7838047 0.8747865 -1.0659786 --2.2445498 -0.5488076 -0.42898977 0.6916364 -1.6542299 -0.9966279 -0.38244298 1.6954672 - -(1,2,.,.) = -0.43478605 -0.6678534 1.9530942 -0.5209587 -0.12899925 0.20572199 2.0359943 0.55223215 -0.65247816 0.8792108 -0.38860792 0.48663738 --1.0084358 0.31141177 0.69208467 0.48385203 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2x4x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = --0.95696485 NaN NaN NaN -NaN NaN -0.13377543 NaN -NaN NaN NaN -0.36869493 -0.5033356 NaN NaN 0.5279584 - -(1,2,.,.) = --0.83290124 NaN 0.6694149 NaN --2.0479486 -1.5812296 0.7109843 -0.5937868 --0.4269776 -0.12873057 NaN -0.720236 -NaN -1.1666392 -0.36804697 -0.72597617 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2x4x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import Log -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Log(input_shape=(2, 4, 4))) -input = np.random.random([1, 2, 4, 4]) -output = model.forward(input) -``` -Input is: -```python -[[[[0.90127539, 0.9861594 , 0.04722941, 0.63719453], - [0.46529477, 0.81511804, 0.24435558, 0.45003562], - [0.15170845, 0.35157662, 0.0925214 , 0.63852947], - [0.27817508, 0.42572846, 0.44363004, 0.03536394]], - - [[0.65027784, 0.00429838, 0.07434429, 0.18653305], - [0.19659183, 0.66647529, 0.77821197, 0.65894478], - [0.28212032, 0.52307663, 0.09589939, 0.71547588], - [0.84344158, 0.25291738, 0.52145649, 0.82982377]]]] -``` -Output is -```python -[[[[-0.10394441, -0.01393729, -3.0527387 , -0.45068032], - [-0.76508415, -0.20442237, -1.4091308 , -0.79842854], - [-1.8857948 , -1.0453277 , -2.3803153 , -0.44858742], - [-1.2795045 , -0.85395354, -0.8127643 , -3.3420627 ]], - - [[-0.43035555, -5.4495163 , -2.5990484 , -1.6791469 ], - [-1.6266255 , -0.4057522 , -0.25075635, -0.41711554], - [-1.2654216 , -0.64802724, -2.3444557 , -0.33480743], - [-0.1702646 , -1.3746924 , -0.6511295 , -0.1865419 ]]]] -``` - ---- -### 7.14 Identity -Identity just return the input to output. - -It's useful in same parallel container to get an origin input. - -**Scala:** -```scala -Identity(inputShape = null) -``` -**Python:** -```python -Identity(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.layers.Identity -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Identity[Float](inputShape = Shape(4, 4))) -val input = Tensor[Float](3, 4, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = -1.9601166 -0.86010313 0.0023731247 -0.81219757 -1.1469674 -1.5375912 -1.5348053 -0.34829113 --1.236773 -0.7183283 -0.89256984 0.8605067 -0.7937664 0.52992857 -1.6157389 0.36134166 - -(2,.,.) = --0.44434744 -0.23848957 -0.01632014 -0.58109635 --0.19856784 -2.3421717 -0.5868049 -0.76775354 -0.80254126 1.78778 -1.1835604 1.4489703 -0.8731402 0.8906672 0.2800079 -0.6715317 - -(3,.,.) = -1.4093032 2.358169 -1.4620789 1.1904576 --0.18263042 -0.31869793 2.01061 1.2159953 --0.5801479 1.2949371 -0.7510707 -1.0707517 -0.30815956 -1.161963 -0.26964024 -0.4759499 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 3x4x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -1.9601166 -0.86010313 0.0023731247 -0.81219757 -1.1469674 -1.5375912 -1.5348053 -0.34829113 --1.236773 -0.7183283 -0.89256984 0.8605067 -0.7937664 0.52992857 -1.6157389 0.36134166 - -(2,.,.) = --0.44434744 -0.23848957 -0.01632014 -0.58109635 --0.19856784 -2.3421717 -0.5868049 -0.76775354 -0.80254126 1.78778 -1.1835604 1.4489703 -0.8731402 0.8906672 0.2800079 -0.6715317 - -(3,.,.) = -1.4093032 2.358169 -1.4620789 1.1904576 --0.18263042 -0.31869793 2.01061 1.2159953 --0.5801479 1.2949371 -0.7510707 -1.0707517 -0.30815956 -1.161963 -0.26964024 -0.4759499 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 3x4x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import Identity -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Identity(input_shape=(4, 4))) -input = np.random.random([3, 4, 4]) -output = model.forward(input) -``` -Input is: -```python -[[[0.36751123, 0.92287101, 0.73894405, 0.33699379], - [0.69405782, 0.9653215 , 0.2617223 , 0.68205229], - [0.71455325, 0.99419333, 0.90886495, 0.10232991], - [0.1644055 , 0.30013138, 0.98921154, 0.26803146]], - - [[0.35898357, 0.72067882, 0.13236563, 0.71935521], - [0.30865626, 0.71098844, 0.86718946, 0.12531168], - [0.84916882, 0.84221518, 0.52186664, 0.87239729], - [0.50637899, 0.10890469, 0.86832705, 0.93581179]], - - [[0.19640105, 0.09341008, 0.12043328, 0.09261859], - [0.66019486, 0.07251262, 0.80929761, 0.39094486], - [0.63027391, 0.39537796, 0.55578905, 0.53933265], - [0.13885559, 0.56695373, 0.17036027, 0.4577097 ]]] -``` -Output is -```python -[[[0.36751124, 0.922871 , 0.73894405, 0.33699378], - [0.6940578 , 0.9653215 , 0.2617223 , 0.6820523 ], - [0.71455324, 0.9941933 , 0.908865 , 0.10232991], - [0.1644055 , 0.30013138, 0.98921156, 0.26803148]], - - [[0.35898358, 0.7206788 , 0.13236563, 0.7193552 ], - [0.30865628, 0.71098846, 0.86718947, 0.12531169], - [0.84916884, 0.8422152 , 0.5218666 , 0.8723973 ], - [0.506379 , 0.10890469, 0.868327 , 0.9358118 ]], - - [[0.19640104, 0.09341008, 0.12043328, 0.09261858], - [0.6601949 , 0.07251262, 0.8092976 , 0.39094487], - [0.63027394, 0.39537796, 0.55578905, 0.5393326 ], - [0.13885559, 0.5669537 , 0.17036027, 0.4577097 ]]] -``` - ---- -### 7.15 Select -Select an index of the input in the given dim and return the subset part. - -The batch dimension needs to be unchanged. - -For example, if input is: - -[[1, 2, 3], - [4, 5, 6]] - -Select(1, 1) will give output [2 5] - -Select(1, -1) will give output [3 6] - -**Scala:** -```scala -Select(dim, index, inputShape = null) -``` -**Python:** -```python -Select(dim, index, input_shape=None, name=None) -``` - -**Parameters:** - -* `dim`: The dimension to select. 0-based index. Cannot select the batch dimension. -1 means the last dimension of the input. -* `index`: The index of the dimension to be selected. 0-based index. -1 means the last dimension of the input. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Select -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Select[Float](1, 2, inputShape = Shape(3, 1, 3))) -val input = Tensor[Float](1, 3, 1, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = --0.67646945 -0.5485965 -0.11103154 -(1,2,.,.) = --0.13488655 0.43843046 -0.04482145 -(1,3,.,.) = --0.18094881 0.19431554 -1.7624844 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x3x1x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = --0.18094881 0.19431554 -1.7624844 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 1x1x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import Select -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(Select(1, 2, input_shape=(3, 1, 3))) -input = np.random.random([1, 3, 1, 3]) -output = model.forward(input) -``` -Input is: -```python -array([[[[0.53306099, 0.95147881, 0.15222129]], - [[0.89604861, 0.90160974, 0.5230576 ]], - [[0.70779386, 0.14438568, 0.37601195]]]]) -``` -Output is: -```python -array([[[0.7077939 , 0.14438568, 0.37601194]]], dtype=float32) -``` - ---- -### 7.16 Dense -A densely-connected NN layer. - -The most common input is 2D. - -**Scala:** -```scala -Dense(outputDim, init = "glorot_uniform", activation = null, wRegularizer = null, bRegularizer = null, bias = true, inputShape = null) -``` -**Python:** -```python -Dense(output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, name=None) -``` - -**Parameters:** - -* `outputDim`: The size of the output dimension. -* `init`: Initialization method for the weights of the layer. Default is Xavier.You can also pass in corresponding string representations such as 'glorot_uniform' or 'normal', etc. for simple init methods in the factory method. -* `activation`: Activation function to use. Default is null.You can also pass in corresponding string representations such as 'relu'or 'sigmoid', etc. for simple activations in the factory method. -* `wRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the input weights matrices. Default is null. -* `bRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the bias. Default is null. -* `bias`: Whether to include a bias (i.e. make the layer affine rather than linear). Default is true. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Dense -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Dense[Float](5, activation = "relu", inputShape = Shape(4))) -val input = Tensor[Float](2, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -1.4289935 -1.7659454 -0.08306135 -1.0153456 -1.0191492 0.37392816 1.3076705 -0.19495767 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -0.5421522 0.49008092 0.0 0.0 0.0 -0.07940009 0.0 0.12953377 0.0 0.0 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x5] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import Dense -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(Dense(5, activation="relu", input_shape=(4, ))) -input = np.random.random([2, 4]) -output = model.forward(input) -``` -Input is: -```python -array([[0.64593485, 0.67393322, 0.72505368, 0.04654095], - [0.19430753, 0.47800889, 0.00743648, 0.6412403 ]]) -``` -Output is -```python -array([[0. , 0. , 1.2698183 , 0. , 0.10656227], - [0. , 0. , 0.6236721 , 0.00299606, 0.29664695]], - dtype=float32) -``` - ---- -### 7.17 Negative -Computes the negative value of each element of the input. - -**Scala:** -```scala -Negative(inputShape = null) -``` -**Python:** -```python -Negative(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Negative -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Negative[Float](inputShape = Shape(2, 3))) -val input = Tensor[Float](2, 2, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = -1.031705 -0.5723963 1.998631 --0.32908052 2.4069138 -2.4111257 -(2,.,.) = -0.5355049 -1.4404331 -0.38116863 --0.45641592 -1.1485358 0.94766915 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = --1.031705 0.5723963 -1.998631 -0.32908052 -2.4069138 2.4111257 -(2,.,.) = --0.5355049 1.4404331 0.38116863 -0.45641592 1.1485358 -0.94766915 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import Negative -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(Negative(input_shape=(2, 3))) -input = np.random.random([2, 2, 3]) -output = model.forward(input) -``` -Input is: -```python -array([[[0.39261261, 0.03164615, 0.32179116], - [0.11969367, 0.61610712, 0.42573733]], - [[0.36794656, 0.90912174, 0.540356 ], - [0.42667627, 0.04154093, 0.84692964]]]) -``` -Output is -```python -array([[[-0.3926126 , -0.03164615, -0.32179114], - [-0.11969367, -0.6161071 , -0.42573732]], - [[-0.36794657, -0.90912175, -0.540356 ], - [-0.42667627, -0.04154094, -0.84692967]]], dtype=float32) -``` - ---- -### 7.18 CAdd -This layer has a bias with given size. - -The bias will be added element-wise to the input. - -If the element number of the bias matches the input, a simple element-wise addition will be done. - -Or the bias will be expanded to the same size of the input. - -The expand means repeat on unmatched singleton dimension (if some unmatched dimension isn't a singleton dimension, an error will be raised). - -**Scala:** -```scala -CAdd(size, bRegularizer = null, inputShape = null) -``` -**Python:** -```python -CAdd(size, b_regularizer=None, input_shape=None, name=None) -``` - -**Parameters:** - -* `size`: the size of the bias -* `bRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the bias. Default is null. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.CAdd -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(CAdd[Float](Array(2, 3), inputShape = Shape(2, 3))) -val input = Tensor[Float](2, 2, 3).rand() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = -0.2183351 0.32434112 0.89350265 -0.3348259 0.78677046 0.24054797 -(2,.,.) = -0.9945844 0.72363794 0.7737936 -0.05522544 0.3517818 0.7417069 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -0.1358028 0.6956667 1.0837181 -0.6767027 0.7955346 0.5063505 -(2,.,.) = -0.9120521 1.0949634 0.96400905 -0.3971022 0.36054593 1.0075095 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import CAdd -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(CAdd([2, 1], input_shape=(2, 3))) -input = np.random.rand(2, 2, 3) -output = model.forward(input) -``` -Input is: -```python -array([[[0.4122004 , 0.73289359, 0.11500016], - [0.26974491, 0.32166632, 0.91408442]], - [[0.66824327, 0.80271314, 0.75981145], - [0.39271431, 0.07312566, 0.4966805 ]]]) -``` -Output is -```python -array([[[ 0.06560206, 0.38629526, -0.23159817], - [ 0.44287407, 0.4947955 , 1.0872136 ]], - [[ 0.32164496, 0.45611483, 0.41321313], - [ 0.56584346, 0.24625483, 0.6698097 ]]], dtype=float32) -``` - ---- -### 7.19 RepeatVector -Repeats the input n times. - -The input of this layer should be 2D, i.e. (num_samples, features). -The output of thi layer should be 3D, i.e. (num_samples, n, features). - -**Scala:** -```scala -RepeatVector(n, inputShape = null) -``` -**Python:** -```python -RepeatVector(n, input_shape=None, name=None) -``` - -**Parameters:** - -* `n`: Repetition factor. Integer. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. -* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.RepeatVector -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(RepeatVector[Float](4, inputShape = Shape(3))) -val input = Tensor[Float](2, 3).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = --0.31839952 -0.3495366 0.542486 --0.54981124 -0.8428188 0.8225184 -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = --0.31839952 -0.3495366 0.542486 --0.31839952 -0.3495366 0.542486 --0.31839952 -0.3495366 0.542486 --0.31839952 -0.3495366 0.542486 - -(2,.,.) = --0.54981124 -0.8428188 0.8225184 --0.54981124 -0.8428188 0.8225184 --0.54981124 -0.8428188 0.8225184 --0.54981124 -0.8428188 0.8225184 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x4x3] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.layers import RepeatVector -from bigdl.dllib.keras.models import Sequential - -model = Sequential() -model.add(RepeatVector(4, input_shape=(3, ))) -input = np.random.random([2, 3]) -output = model.forward(input) -``` -Input is: -```python -array([[ 0.90715922, 0.54594769, 0.53952404], - [ 0.08989831, 0.07265549, 0.45830114]]) -``` -Output is -```python -array([[[ 0.90715921, 0.54594767, 0.53952402], - [ 0.90715921, 0.54594767, 0.53952402], - [ 0.90715921, 0.54594767, 0.53952402], - [ 0.90715921, 0.54594767, 0.53952402]], - - [[ 0.08989831, 0.07265549, 0.45830116], - [ 0.08989831, 0.07265549, 0.45830116], - [ 0.08989831, 0.07265549, 0.45830116], - [ 0.08989831, 0.07265549, 0.45830116]]], dtype=float32) -``` ---- -### 7.20 GaussianSampler -Takes {mean, log_variance} as input and samples from the Gaussian distribution. - -**Scala:** -```scala -GaussianSampler(inputShape = null) -``` -**Python:** -```python -GaussianSampler(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object that consists of two identical Single Shape. For Python API, it should be a list of two identical shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.GaussianSampler -import com.intel.analytics.bigdl.utils.{Shape, MultiShape, T} -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -val shape1 = Shape(2, 3) -val shape2 = Shape(2, 3) -model.add(GaussianSampler[Float](inputShape = MultiShape(List(shape1,shape2)))) -val input1 = Tensor[Float](2, 2, 3).rand(0, 1) -val input2 = Tensor[Float](2, 2, 3).rand(0, 1) -val input = T(1 -> input1, 2 -> input2) -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.utils.Table = - { - 2: (1,.,.) = - 0.9996127 0.8964211 0.7424038 - 0.40628982 0.37035564 0.20108517 - - (2,.,.) = - 0.6974727 0.60202897 0.1535999 - 0.012422224 0.5993025 0.96206 - - [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] - 1: (1,.,.) = - 0.21060324 0.576583 0.21633287 - 0.1484059 0.2730577 0.25317845 - - (2,.,.) = - 0.58513683 0.58095694 0.18811373 - 0.7029449 0.41235915 0.44636542 - - [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] - } -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -1.5258198 1.9536011 -1.8591263 --1.0618867 -0.751225 0.35412917 - -(2,.,.) = -1.3334517 -0.60312974 0.7324476 -0.09502721 0.8094909 0.44807082 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.models import Sequential -from bigdl.dllib.keras.layers import GaussianSampler - -model = Sequential() -model.add(GaussianSampler(input_shape=[(3,),(3,)])) -input1 = np.random.random([2, 3]) -input2 = np.random.random([2, 3]) -input = [input1, input2] -output = model.forward(input) -``` -Input is: -```python -[[[0.79941342, 0.87462822, 0.9516901 ], - [0.20111287, 0.54634077, 0.83614511]], - - [[0.31886989, 0.22829382, 0.84355419], - [0.51186641, 0.28043938, 0.29440057]]] -``` -Output is -```python -[[ 0.71405387 2.2944303 -0.41778684] - [ 0.84234 2.3337283 -0.18952972]] -``` - ---- -### 7.21 Exp -Applies element-wise exp to the input. - -When you use this layer as the first layer of a model, you need to provide the argument inputShape (a Single Shape, does not include the batch dimension). - -**Scala:** -```scala -Exp(inputShape = null) -``` -**Python:** -```python -Exp(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object that consists of two identical Single Shape. For Python API, it should be a list of two identical shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Exp -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Exp[Float](inputShape = Shape(2, 3, 4))) -val input = Tensor[Float](2, 2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = --1.5841372 -0.13795324 -2.144475 0.09272669 -1.055668 -1.2310301 1.2145554 -0.6073714 -0.9296467 0.2923885 1.3364213 0.1652137 - -(1,2,.,.) = -0.2099718 -0.3856573 -0.92586 -0.5317779 -0.6618383 -0.9677452 -1.5014665 -0.35464883 -2.045924 -0.317644 -1.812726 0.95438373 - -(2,1,.,.) = --0.4536791 -0.34785584 1.6424289 -0.07981159 --0.8022624 -0.4211059 0.3461831 1.9598864 --0.84695745 -0.6115283 0.7729755 2.3077402 - -(2,2,.,.) = --0.08438411 -0.908458 0.6688936 -0.7292123 --0.26337254 0.55425745 -0.14925817 -0.010179609 --0.62562865 -1.0517743 -0.23839666 -1.144982 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.20512469 0.8711394 0.11712951 1.0971619 -2.8738942 0.29199165 3.3687959 0.544781 -2.533614 1.3396233 3.8054006 1.1796452 - -(1,2,.,.) = -1.2336433 0.6800035 0.39619055 0.5875594 -1.9383523 0.37993878 0.22280318 0.7014197 -7.7363033 0.7278619 0.16320862 2.5970695 - -(2,1,.,.) = -0.63528657 0.70620066 5.167706 0.92329025 -0.44831353 0.6563206 1.4136615 7.0985208 -0.42871734 0.5425211 2.1662023 10.051684 - -(2,2,.,.) = -0.9190782 0.4031454 1.9520763 0.48228875 -0.76845556 1.740648 0.8613467 0.98987204 -0.53492504 0.34931743 0.7878901 0.31822965 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.models import Sequential -from bigdl.dllib.keras.layers import Exp - -model = Sequential() -model.add(Exp(input_shape=(2, 3, 4))) -input = np.random.random([2, 2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -[[[[0.93104587 0.94000338 0.84870765 0.98645553] - [0.83708846 0.33375541 0.50119834 0.24879265] - [0.51966475 0.84514791 0.15496452 0.61538968]] - - [[0.57250337 0.42520832 0.94850757 0.54317573] - [0.64228691 0.9904079 0.01008592 0.51365217] - [0.78640595 0.7717037 0.51277595 0.24245034]]] - - - [[[0.82184752 0.92537331 0.20632728 0.47539445] - [0.44604637 0.1507692 0.5437313 0.2074501 ] - [0.93661363 0.93962609 0.29230559 0.74850958]] - - [[0.11659768 0.76177132 0.33194573 0.20695088] - [0.49636212 0.85987328 0.49767861 0.96774006] - [0.67669121 0.15542122 0.69981032 0.3349874 ]]]] -``` -Output is -```python -[[[[2.5371614 2.5599902 2.3366253 2.6817122] - [2.3096325 1.3962016 1.6506982 1.2824761] - [1.6814638 2.3283222 1.1676165 1.8503776]] - - [[1.7726992 1.5299091 2.5818534 1.721465 ] - [1.9008229 2.6923325 1.010137 1.6713842] - [2.1954916 2.163449 1.6699204 1.2743679]]] - - - [[[2.2746985 2.52281 1.2291554 1.6086487] - [1.5621239 1.1627283 1.7224218 1.2305363] - [2.551327 2.5590243 1.3395122 2.1138473]] - - [[1.1236672 2.1420672 1.3936772 1.2299222] - [1.6427343 2.3628614 1.6448984 2.6319895] - [1.9673574 1.16815 2.0133708 1.3979228]]]] -``` - ---- -### 7.22 Square -Applies an element-wise square operation to the input. - -When you use this layer as the first layer of a model, you need to provide the argument inputShape (a Single Shape, does not include the batch dimension). - -**Scala:** -```scala -Square(inputShape = null) -``` -**Python:** -```python -Square(input_shape=None, name=None) -``` - -**Parameters:** - -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object that consists of two identical Single Shape. For Python API, it should be a list of two identical shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Square -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Square[Float](inputShape = Shape(2, 3, 4))) -val input = Tensor[Float](2, 2, 3, 4).randn() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = --0.108013034 1.8879265 1.2232096 -1.5076439 -1.4895755 -0.37966672 -0.34892964 0.15224025 --0.9296686 -1.1523775 0.14153497 -0.26954007 - -(1,2,.,.) = --1.0875931 2.190617 -0.6903083 1.0039362 --0.1275677 -1.1096588 0.37359753 -0.17367937 -0.23349741 0.14639114 -0.2330162 0.5343827 - -(2,1,.,.) = -0.3222191 0.21463287 -1.0157064 -0.22627507 -1.1714277 0.43371263 1.069315 0.5122436 -0.1958086 -1.4601041 2.5394423 -0.470833 - -(2,2,.,.) = --0.38708544 -0.951611 -0.37234613 0.26813275 -1.9477026 0.32779223 -1.2308712 -2.2376378 -0.19652915 0.3304719 -1.7674786 -0.86961496 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.011666816 3.5642662 1.4962418 2.2729902 -2.218835 0.14414681 0.1217519 0.023177093 -0.86428374 1.3279738 0.020032147 0.07265185 - -(1,2,.,.) = -1.1828587 4.7988033 0.47652552 1.0078878 -0.016273517 1.2313428 0.13957511 0.030164523 -0.05452104 0.021430366 0.054296546 0.28556487 - -(2,1,.,.) = -0.10382515 0.046067268 1.0316595 0.05120041 -1.3722429 0.18810664 1.1434345 0.26239353 -0.038341008 2.131904 6.448767 0.22168371 - -(2,2,.,.) = -0.14983514 0.9055635 0.13864164 0.07189517 -3.7935455 0.10744774 1.5150439 5.007023 -0.038623706 0.109211676 3.1239805 0.7562302 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` - -**Python example:** -```python -import numpy as np -from bigdl.dllib.keras.models import Sequential -from bigdl.dllib.keras.layers import Square - -model = Sequential() -model.add(Square(input_shape=(2, 3, 4))) -input = np.random.random([2, 2, 3, 4]) -output = model.forward(input) -``` -Input is: -```python -[[[[0.8708819 0.2698243 0.55854849 0.71699472] - [0.66647234 0.72310216 0.8082119 0.66566951] - [0.6714764 0.61394108 0.35063125 0.60473593]] - - [[0.37993365 0.64222557 0.96762005 0.18931697] - [0.00529722 0.99133455 0.09786619 0.28988077] - [0.60052911 0.83712995 0.59847519 0.54361243]]] - - - [[[0.32832672 0.83316023 0.41272485 0.01963383] - [0.89593955 0.73433713 0.67529323 0.69711912] - [0.81251711 0.56755577 0.31958151 0.09795917]] - - [[0.46465895 0.22818875 0.31505317 0.41912166] - [0.87865447 0.3799063 0.091204 0.68144165] - [0.88274284 0.70479132 0.32074672 0.71771481]]]] -``` -Output is -```python -[[[[7.5843531e-01 7.2805151e-02 3.1197643e-01 5.1408142e-01] - [4.4418535e-01 5.2287674e-01 6.5320653e-01 4.4311589e-01] - [4.5088059e-01 3.7692365e-01 1.2294226e-01 3.6570552e-01]] - - [[1.4434958e-01 4.1245368e-01 9.3628860e-01 3.5840917e-02] - [2.8060573e-05 9.8274422e-01 9.5777912e-03 8.4030852e-02] - [3.6063525e-01 7.0078653e-01 3.5817260e-01 2.9551446e-01]]] - - - [[[1.0779844e-01 6.9415593e-01 1.7034180e-01 3.8548734e-04] - [8.0270761e-01 5.3925103e-01 4.5602092e-01 4.8597506e-01] - [6.6018403e-01 3.2211956e-01 1.0213234e-01 9.5959986e-03]] - - [[2.1590793e-01 5.2070107e-02 9.9258497e-02 1.7566296e-01] - [7.7203369e-01 1.4432879e-01 8.3181690e-03 4.6436274e-01] - [7.7923489e-01 4.9673077e-01 1.0287846e-01 5.1511449e-01]]]] -``` - ---- -### 7.23 Power -Applies an element-wise power operation with scale and shift to the input. - -f(x) = (shift + scale * x)^power^ - -```scala -Power(power, scale = 1, shift = 0, inputShape = null) -``` -**Python:** -```python -Power(power, scale=1, shift=0, input_shape=None, name=None) -``` - -**Parameters:** - -* `power`: The exponent -* `scale`: The scale parameter. Default is 1. -* `shift`: The shift parameter. Default is 0. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Power -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Power[Float](2, inputShape = Shape(2, 3))) -val input = Tensor[Float](2, 2, 3).rand() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = -0.24691099 0.7588585 0.5785183 -0.10356348 0.2252714 0.3129436 - -(2,.,.) = -0.6277785 0.75136995 0.044648796 -0.46396527 0.9793776 0.92727077 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -0.060965035 0.5758662 0.3346834 -0.010725395 0.050747205 0.0979337 - -(2,.,.) = -0.39410582 0.5645568 0.001993515 -0.21526377 0.95918053 0.8598311 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import Power -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(Power(2, input_shape=(2, 3))) -input = np.random.rand(2, 2, 3) -output = model.forward(input) -``` -Input is: -```python -array([[[0.5300817 , 0.18128031, 0.19534253], - [0.28380639, 0.78365165, 0.6893 ]], - - [[0.05574091, 0.400077 , 0.77051193], - [0.033559 , 0.61051396, 0.13970227]]]) -``` -Output is -```python -array([[[0.2809866 , 0.03286255, 0.03815871], - [0.08054607, 0.61410993, 0.4751345 ]], - - [[0.00310705, 0.16006161, 0.5936886 ], - [0.00112621, 0.37272733, 0.01951673]]], dtype=float32) -``` - ---- -### 7.24 AddConstant -Add a (non-learnable) scalar constant to the input. - -```scala -AddConstant(constant, inputShape = null) -``` -**Python:** -```python -AddConstant(constant, input_shape=None, name=None) -``` - -**Parameters:** - -* `constant`: The scalar constant to be added. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.AddConstant -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(AddConstant[Float](1, inputShape = Shape(2, 3))) -val input = Tensor[Float](2, 2, 3).rand() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,.,.) = -0.5658301 0.3508225 0.4012322 -0.1941942 0.18934165 0.6909284 - -(2,.,.) = -0.5985211 0.5485885 0.778548 -0.16745302 0.10363362 0.92185616 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,.,.) = -1.5658301 1.3508224 1.4012322 -1.1941942 1.1893417 1.6909285 - -(2,.,.) = -1.5985211 1.5485885 1.778548 -1.167453 1.1036336 1.9218562 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import AddConstant -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(AddConstant(1, input_shape=(2, 3))) -input = np.random.rand(2, 2, 3) -output = model.forward(input) -``` -Input is: -```python -array([[[0.71730919, 0.07752598, 0.10448237], - [0.52319608, 0.38668494, 0.19588814]], - - [[0.15496092, 0.48405899, 0.41441248], - [0.13792111, 0.7523953 , 0.55991187]]]) -``` -Output is -```python -array([[[1.7173092, 1.077526 , 1.1044824], - [1.5231961, 1.3866849, 1.1958882]], - - [[1.1549609, 1.484059 , 1.4144125], - [1.1379211, 1.7523953, 1.5599118]]], dtype=float32) -``` - ---- -### 7.25 Narrow -Narrow the input with the number of dimensions not being reduced. - -The batch dimension needs to be unchanged. - -For example, if input is: - -[[1 2 3], - [4 5 6]] - -Narrow(1, 1, 2) will give output - -[[2 3], - [5 6]] - -Narrow(1, 2, -1) will give output - -[3, - 6] - -```scala -Narrow(dim, offset, length = 1, inputShape = null) -``` -**Python:** -```python -Narrow(dim, offset, length=1, input_shape=None, name=None) -``` - -**Parameters:** - -* `dim`: The dimension to narrow. 0-based index. Cannot narrow the batch dimension. - -1 means the last dimension of the input. -* `offset`: Non-negative integer. The start index on the given dimension. 0-based index. -* `length`: The length to narrow. Default is 1. - Can use a negative length such as -1 in the case where input size is unknown. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Narrow -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Narrow[Float](1, 1, inputShape = Shape(2, 3, 4))) -val input = Tensor[Float](2, 2, 3, 4).rand() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = -0.13770224 0.63719153 0.7776689 0.46612367 -0.9026256 0.11982094 0.8282868 0.05095969 -0.889799 0.6386537 0.35438475 0.298043 - -(1,2,.,.) = -0.5029727 0.20103335 0.20150806 0.06437344 -0.2255908 0.5388977 0.59737855 0.5210477 -0.4055072 0.11848069 0.7118382 0.9796308 - -(2,1,.,.) = -0.63957494 0.1921936 0.7749439 0.19744827 -0.91683346 0.16140814 0.9753973 0.8161283 -0.8481694 0.8802563 0.1233245 0.5732614 - -(2,2,.,.) = -0.275001 0.35905758 0.15939762 0.09233412 -0.16610192 0.032060683 0.37298614 0.48936844 -0.031097537 0.82767457 0.10246291 0.9951448 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.5029727 0.20103335 0.20150806 0.06437344 -0.2255908 0.5388977 0.59737855 0.5210477 -0.4055072 0.11848069 0.7118382 0.9796308 - -(2,1,.,.) = -0.275001 0.35905758 0.15939762 0.09233412 -0.16610192 0.032060683 0.37298614 0.48936844 -0.031097537 0.82767457 0.10246291 0.9951448 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x1x3x4] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import Narrow -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(Narrow(1, 1, input_shape=(2, 3, 4))) -input = np.random.rand(2, 2, 3, 4) -output = model.forward(input) -``` -Input is: -```python -array([[[[0.74305305, 0.33925069, 0.31289333, 0.43703923], - [0.28316902, 0.3004414 , 0.40298034, 0.37476436], - [0.18825825, 0.38979411, 0.32963262, 0.37783457]], - - [[0.14824117, 0.43532988, 0.57077087, 0.91535978], - [0.46375725, 0.90511296, 0.18859044, 0.92820822], - [0.13675737, 0.48270908, 0.04260755, 0.97255687]]], - [[[0.4836805 , 0.45262542, 0.7233705 , 0.63486529], - [0.07472717, 0.5715716 , 0.57029986, 0.26475783], - [0.56757079, 0.27602746, 0.45799196, 0.74420842]], - - [[0.89048761, 0.08280716, 0.99030481, 0.35956427], - [0.70802689, 0.14425212, 0.08320864, 0.82271697], - [0.6915224 , 0.70490768, 0.41218963, 0.37024863]]]]) -``` -Output is -```python -array([[[[0.14824118, 0.43532988, 0.57077086, 0.9153598 ], - [0.46375725, 0.905113 , 0.18859044, 0.92820823], - [0.13675737, 0.48270908, 0.04260755, 0.9725569 ]]], - - [[[0.8904876 , 0.08280716, 0.9903048 , 0.35956427], - [0.7080269 , 0.14425212, 0.08320864, 0.82271695], - [0.6915224 , 0.70490766, 0.41218963, 0.37024862]]]], - dtype=float32) -``` - ---- -### 7.26 Permute -Permutes the dimensions of the input according to a given pattern. - -Useful for connecting RNNs and convnets together. - -```scala -Permute(dims, inputShape = null) -``` -**Python:** -```python -Permute(dims, input_shape=None, name=None) -``` - -**Parameters:** - -* `dims`: Int array. Permutation pattern, does not include the batch dimension. - Indexing starts at 1. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.Permute -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential[Float]() -model.add(Permute[Float](Array(2, 1, 3), inputShape = Shape(2, 2, 3))) -val input = Tensor[Float](2, 2, 2, 3).rand() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = -0.8451549 0.06361471 0.7324815 -0.31086245 0.21210302 0.35112163 - -(1,2,.,.) = -0.61466074 0.50173014 0.8759959 -0.19090249 0.671227 0.73089105 -(2,1,.,.) = -0.47867084 0.9341955 0.063592255 -0.24063066 0.502274 0.9114748 -(2,2,.,.) = -0.93335986 0.25173688 0.88615775 -0.5394321 0.330763 0.89036304 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x2x3] -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.8451549 0.06361471 0.7324815 -0.61466074 0.50173014 0.8759959 - -(1,2,.,.) = -0.31086245 0.21210302 0.35112163 -0.19090249 0.671227 0.73089105 -(2,1,.,.) = -0.47867084 0.9341955 0.063592255 -0.93335986 0.25173688 0.88615775 -(2,2,.,.) = -0.24063066 0.502274 0.9114748 -0.5394321 0.330763 0.89036304 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x2x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import Permute -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(Permute((2, 1, 3), input_shape=(2, 2, 3))) -input = np.random.rand(2, 2, 2, 3) -output = model.forward(input) -``` -Input is: -```python -array([[[[0.14016896, 0.7275626 , 0.79087092], - [0.57259566, 0.97387138, 0.70001999]], - - [[0.9232002 , 0.07644555, 0.24705828], - [0.17257354, 0.93951155, 0.46183983]]], - [[[0.79432476, 0.64299062, 0.33959594], - [0.58608318, 0.338014 , 0.92602687]], - - [[0.32638575, 0.69032582, 0.25168083], - [0.46813027, 0.95118373, 0.13145026]]]]) -``` -Output is -```python -array([[[[0.14016896, 0.7275626 , 0.7908709 ], - [0.9232002 , 0.07644555, 0.24705827]], - - [[0.57259566, 0.97387135, 0.70002 ], - [0.17257354, 0.93951154, 0.46183982]]], - [[[0.79432476, 0.64299065, 0.33959594], - [0.32638577, 0.6903258 , 0.25168082]], - [[0.5860832 , 0.338014 , 0.9260269 ], - [0.46813026, 0.95118374, 0.13145027]]]], dtype=float32) -``` ---- -### 7.27 ResizeBilinear -Resize the input image with bilinear interpolation. The input image must be a float tensor with NHWC or NCHW layout. - -```scala -ResizeBilinear(outputHeight, outputWidth, alignCorners = false, dimOrdering = "th", inputShape = null) -``` -**Python:** -```python -ResizeBilinear(output_height, output_width, align_corner=False, dim_ordering="th", input_shape=(2, 3, 5, 7), name=None) -``` - -**Parameters:** - -* `outputHeight`: output height -* `outputWidth`: output width -* `alignCorners`: align corner or not -* `dimOrdering`: Format of input data. Either DataFormat.NCHW (dimOrdering='th') or DataFormat.NHWC (dimOrdering='tf'). Default is NCHW. -* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. - -**Scala example:** -```scala -import com.intel.analytics.bigdl.dllib.keras.models.Sequential -import com.intel.analytics.bigdl.dllib.keras.layers.ResizeBilinear -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.tensor.Tensor - -val model = Sequential() -model.add(ResizeBilinear[Float](2, 3, inputShape = Shape(2, 3, 5))) -val input = Tensor[Float](2, 2, 3, 5).rand() -val output = model.forward(input) -``` -Input is: -```scala -input: com.intel.analytics.bigdl.tensor.Tensor[Float] = -(1,1,.,.) = -0.6991891 0.007127314 0.73871046 0.95916307 0.9433856 -0.41275907 0.37573513 0.99193203 0.06930728 0.5922364 -0.024281504 0.2592453 0.3898136 0.6635241 0.85888565 - -(1,2,.,.) = -0.38028112 0.43709648 0.62538666 0.8468501 0.6445014 -0.45252413 0.48801896 0.59471387 0.013207023 0.3567462 -0.85187584 0.49279585 0.7973665 0.81287366 0.07852263 - -(2,1,.,.) = -0.1452374 0.6140467 0.36384684 0.066476084 0.96101314 -0.54862195 0.66091377 0.86857307 0.6844842 0.7368217 -0.25342992 0.71737933 0.12789607 0.21691357 0.7543404 - -(2,2,.,.) = -0.79176855 0.1204049 0.58971256 0.115073755 0.10459962 -0.5225398 0.742363 0.7612815 0.9881919 0.13359445 -0.9026869 0.13972941 0.92064524 0.9435532 0.5502235 - -[com.intel.analytics.bigdl.tensor.DenseTensor of... -``` -Output is: -```scala -output: com.intel.analytics.bigdl.nn.abstractnn.Activity = -(1,1,.,.) = -0.6991891 0.4948494 0.9539039 -0.21852028 0.5664119 0.48613077 - -(1,2,.,.) = -0.38028112 0.56262326 0.7794005 -0.6522 0.6274959 0.34790504 - -(2,1,.,.) = -0.1452374 0.4472468 0.36465502 -0.40102595 0.5618719 0.54899293 - -(2,2,.,.) = -0.79176855 0.43327665 0.111582376 -0.71261334 0.70765764 0.75788474 - -[com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x2x3] -``` - -**Python example:** -```python -from bigdl.dllib.keras.layers import ResizeBilinear -from bigdl.dllib.keras.models import Sequential -import numpy as np - -model = Sequential() -model.add(ResizeBilinear(2, 3, input_shape=(2, 3, 5, 5))) -input = np.random.rand(2, 2, 3, 5, 5) -output = model.forward(input) -``` -Input is: -```python -array([[[[0.43790358, 0.41882914, 0.71929122, 0.19673119, 0.36950189], - [0.38808651, 0.34287751, 0.34076998, 0.02581254, 0.42406155], - [0.84648848, 0.18411068, 0.97545126, 0.5468195 , 0.32136674]], - - [[0.32965599, 0.06883324, 0.17350748, 0.01181338, 0.59180775], - [0.24667588, 0.36422516, 0.59648387, 0.48699443, 0.32323264], - [0.67661373, 0.58779956, 0.55286771, 0.59629101, 0.69727522]]], - - - [[[0.09462238, 0.35658325, 0.6787812 , 0.78676645, 0.99019452], - [0.81501527, 0.13348641, 0.71749101, 0.40543351, 0.3959018 ], - [0.608378 , 0.10531177, 0.78000335, 0.51679768, 0.65067605]], - - [[0.12074634, 0.92682843, 0.52227042, 0.98856558, 0.28105255], - [0.78411841, 0.19625097, 0.83108171, 0.03777509, 0.15700493], - [0.95528158, 0.94003855, 0.61092905, 0.68651048, 0.57563719]]]]) -``` -Output is -```python -array([[[[0.43790358, 0.61913717, 0.2543214 ], - [0.6172875 , 0.52657175, 0.3151154 ]], - - [[0.329656 , 0.13861606, 0.20514478], - [0.46164483, 0.541788 , 0.5311798 ]]], - - - [[[0.09462238, 0.57138187, 0.8545758 ], - [0.7116966 , 0.5389645 , 0.48184 ]], - - [[0.12074634, 0.6571231 , 0.752728 ], - [0.86969995, 0.6700518 , 0.36353552]]]], dtype=float32) -``` ---- -## 8. Persistence +## 7. Persistence This section describes how to save and load the Keras-like API. -### 8.1 save +### 7.1 save To save a Keras model, you call the method `saveModel(path)`. **Scala:** @@ -2859,7 +170,7 @@ model.add(Dense(input_shape=(32, ))) model.saveModel("/tmp/seq.model") ``` -### 8.2 load +### 7.2 load To load a saved Keras model, you call the method `load_model(path)`. **Scala:** diff --git a/docs/readthedocs/source/doc/DLlib/Overview/python-getting-started.md b/docs/readthedocs/source/doc/DLlib/Overview/python-getting-started.md deleted file mode 100644 index c03a990c..00000000 --- a/docs/readthedocs/source/doc/DLlib/Overview/python-getting-started.md +++ /dev/null @@ -1,216 +0,0 @@ -# Python DLLib Getting Start Guide - -## 1. Code initialization -```nncontext``` is the main entry for provisioning the dllib program on the underlying cluster (such as K8s or Hadoop cluster), or just on a single laptop. - -It is recommended to initialize `nncontext` at the beginning of your program: -``` -from bigdl.dllib.nncontext import * -sc = init_nncontext() -``` -For more information about ```nncontext```, please refer to [nncontext](https://bigdl.readthedocs.io/en/latest/doc/DLlib/Overview/dllib.html#nn-context) - -## 3. Distributed Data Loading - -#### Using Spark Dataframe APIs -DLlib supports Spark Dataframes as the input to the distributed training, and as -the input/output of the distributed inference. Consequently, the user can easily -process large-scale dataset using Apache Spark, and directly apply AI models on -the distributed (and possibly in-memory) Dataframes without data conversion or serialization - -We create Spark session so we can use Spark API to load and process the data -``` -spark = SQLContext(sc) -``` - -1. We can use Spark API to load the data into Spark DataFrame, eg. read csv file into Spark DataFrame -``` -path = "pima-indians-diabetes.data.csv" -spark.read.csv(path) -``` - -If the feature column for the model is a Spark ML Vector. Please assemble related columns into a Vector and pass it to the model. eg. -``` -from pyspark.ml.feature import VectorAssembler -vecAssembler = VectorAssembler(outputCol="features") -vecAssembler.setInputCols(["num_times_pregrant", "plasma_glucose", "blood_pressure", "skin_fold_thickness", "2-hour_insulin", "body_mass_index", "diabetes_pedigree_function", "age"]) -assemble_df = vecAssembler.transform(df) -assemble_df.withColumn("label", col("class").cast(DoubleType) + lit(1)) -``` - -2. If the training data is image, we can use DLLib api to load image into Spark DataFrame. Eg. -``` -imgPath = "cats_dogs/" -imageDF = NNImageReader.readImages(imgPath, sc) -``` - -It will load the images and generate feature tensors automatically. Also we need generate labels ourselves. eg: -``` -labelDF = imageDF.withColumn("name", getName(col("image"))) \ - .withColumn("label", getLabel(col('name'))) -``` - -Then split the Spark DataFrame into traing part and validation part -``` -(trainingDF, validationDF) = labelDF.randomSplit([0.9, 0.1]) -``` - -## 4. Model Definition - -#### Using Keras-like APIs - -To define a model, you can use the [Keras Style API](https://bigdl.readthedocs.io/en/latest/doc/DLlib/Overview/keras-api.html). -``` -x1 = Input(shape=[8]) -dense1 = Dense(12, activation="relu")(x1) -dense2 = Dense(8, activation="relu")(dense1) -dense3 = Dense(2)(dense2) -dmodel = Model(input=x1, output=dense3) -``` - -After creating the model, you will have to decide which loss function to use in training. - -Now you can use `compile` function of the model to set the loss function, optimization method. -``` -dmodel.compile(optimizer = "adam", loss = "sparse_categorical_crossentropy") -``` - -Now the model is built and ready to train. - -## 5. Distributed Model Training -Now you can use 'fit' begin the training, please set the label columns. Model Evaluation can be performed periodically during a training. -1. If the dataframe is generated using Spark apis, you also need set the feature columns. eg. -``` -model.fit(df, feature_cols=["features"], label_cols=["label"], batch_size=4, nb_epoch=1) -``` -Note: Above model accepts single input(column `features`) and single output(column `label`). - -If your model accepts multiple inputs(eg. column `f1`, `f2`, `f3`), please set the features as below: -``` -model.fit(df, feature_cols=["f1", "f2"], label_cols=["label"], batch_size=4, nb_epoch=1) -``` - -Similarly, if the model accepts multiple outputs(eg. column `label1`, `label2`), please set the label columns as below: -``` -model.fit(df, feature_cols=["features"], label_cols=["l1", "l2"], batch_size=4, nb_epoch=1) -``` - -2. If the dataframe is generated using DLLib `NNImageReader`, we don't need set `feature_cols`, we can set `transform` to config how to process the images before training. Eg. -``` -from bigdl.dllib.feature.image import transforms -transformers = transforms.Compose([ImageResize(50, 50), ImageMirror()]) -model.fit(image_df, label_cols=["label"], batch_size=1, nb_epoch=1, transform=transformers) -``` -For more details about how to use DLLib keras api to train image data, you may want to refer [ImageClassification](https://github.com/intel-analytics/BigDL/tree/main/python/dllib/examples/keras/image_classification.py) - -## 6. Model saving and loading -When training is finished, you may need to save the final model for later use. - -BigDL allows you to save your BigDL model on local filesystem, HDFS, or Amazon s3. -- **save** -``` -modelPath = "/tmp/demo/keras.model" -dmodel.saveModel(modelPath) -``` - -- **load** -``` -loadModel = Model.loadModel(modelPath) -preDF = loadModel.predict(df, feature_cols=["features"], prediction_col="predict") -``` - -You may want to refer [Save/Load](https://bigdl.readthedocs.io/en/latest/doc/DLlib/Overview/keras-api.html#save) - -## 7. Distributed evaluation and inference -After training finishes, you can then use the trained model for prediction or evaluation. - -- **inference** -1. For dataframe generated by Spark API, please set `feature_cols` and `prediction_col` -``` -dmodel.predict(df, feature_cols=["features"], prediction_col="predict") -``` -2. For dataframe generated by `NNImageReader`, please set `prediction_col` and you can set `transform` if needed -``` -model.predict(df, prediction_col="predict", transform=transformers) -``` - -- **evaluation** -Similary for dataframe generated by Spark API, the code is as below: -``` -dmodel.evaluate(df, batch_size=4, feature_cols=["features"], label_cols=["label"]) -``` - -For dataframe generated by `NNImageReader`: -``` -model.evaluate(image_df, batch_size=1, label_cols=["label"], transform=transformers) -``` - -## 8. Checkpointing and resuming training -You can configure periodically taking snapshots of the model. -``` -cpPath = "/tmp/demo/cp" -dmodel.set_checkpoint(cpPath) -``` -You can also set ```over_write``` to ```true``` to enable overwriting any existing snapshot files - -After training stops, you can resume from any saved point. Choose one of the model snapshots to resume (saved in checkpoint path, details see Checkpointing). Use Models.loadModel to load the model snapshot into an model object. -``` -loadModel = Model.loadModel(path) -``` - -## 9. Monitor your training - -- **Tensorboard** - -BigDL provides a convenient way to monitor/visualize your training progress. It writes the statistics collected during training/validation. Saved summary can be viewed via TensorBoard. - -In order to take effect, it needs to be called before fit. -``` -dmodel.set_tensorboard("./", "dllib_demo") -``` -For more details, please refer [visulization](visualization.md) - -## 10. Transfer learning and finetuning - -- **freeze and trainable** -BigDL DLLib supports exclude some layers of model from training. -``` -dmodel.freeze(layer_names) -``` -Layers that match the given names will be freezed. If a layer is freezed, its parameters(weight/bias, if exists) are not changed in training process. - -BigDL DLLib also support unFreeze operations. The parameters for the layers that match the given names will be trained(updated) in training process -``` -dmodel.unFreeze(layer_names) -``` -For more information, you may refer [freeze](freeze.md) - -## 11. Hyperparameter tuning -- **optimizer** - -DLLib supports a list of optimization methods. -For more details, please refer [optimization](optim-Methods.md) - -- **learning rate scheduler** - -DLLib supports a list of learning rate scheduler. -For more details, please refer [lr_scheduler](learningrate-Scheduler.md) - -- **batch size** - -DLLib supports set batch size during training and prediction. We can adjust the batch size to tune the model's accuracy. - -- **regularizer** - -DLLib supports a list of regularizers. -For more details, please refer [regularizer](regularizers.md) - -- **clipping** - -DLLib supports gradient clipping operations. -For more details, please refer [gradient_clip](clipping.md) - -## 12. Running program -``` -python you_app_code.py -``` diff --git a/docs/readthedocs/source/doc/DLlib/Overview/scala-getting-started.md b/docs/readthedocs/source/doc/DLlib/Overview/scala-getting-started.md deleted file mode 100644 index 00c40350..00000000 --- a/docs/readthedocs/source/doc/DLlib/Overview/scala-getting-started.md +++ /dev/null @@ -1,301 +0,0 @@ -# DLLib Getting Start Guide - -## 1. Creating dev environment - -#### Scala project (maven & sbt) - -- **Maven** - -To use BigDL DLLib to build your own deep learning application, you can use maven to create your project and add bigdl-dllib to your dependency. Please add below code to your pom.xml to add BigDL DLLib as your dependency: -``` - - com.intel.analytics.bigdl - bigdl-dllib-spark_2.4.6 - 0.14.0 - -``` - -- **SBT** -``` -libraryDependencies += "com.intel.analytics.bigdl" % "bigdl-dllib-spark_2.4.6" % "0.14.0" -``` -For more information about how to add BigDL dependency, please refer https://bigdl.readthedocs.io/en/latest/doc/UserGuide/scala.html#build-a-scala-project - -#### IDE (Intelij) -Open up IntelliJ and click File => Open - -Navigate to your project. If you have add BigDL DLLib as dependency in your pom.xml. -The IDE will automatically download it from maven and you are able to run your application. - -For more details about how to setup IDE for BigDL project, please refer https://bigdl-project.github.io/master/#ScalaUserGuide/install-build-src/#setup-ide - - -## 2. Code initialization -```NNContext``` is the main entry for provisioning the dllib program on the underlying cluster (such as K8s or Hadoop cluster), or just on a single laptop. - -It is recommended to initialize `NNContext` at the beginning of your program: -``` -import com.intel.analytics.bigdl.dllib.NNContext -import com.intel.analytics.bigdl.dllib.keras.Model -import com.intel.analytics.bigdl.dllib.keras.models.Models -import com.intel.analytics.bigdl.dllib.keras.optimizers.Adam -import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion -import com.intel.analytics.bigdl.dllib.utils.Shape -import com.intel.analytics.bigdl.dllib.keras.layers._ -import com.intel.analytics.bigdl.numeric.NumericFloat -import org.apache.spark.ml.feature.VectorAssembler -import org.apache.spark.sql.SQLContext -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.DoubleType - -val sc = NNContext.initNNContext("dllib_demo") -``` -For more information about ```NNContext```, please refer to [NNContext](https://bigdl.readthedocs.io/en/latest/doc/DLlib/Overview/dllib.html#nn-context) - -## 3. Distributed Data Loading - -#### Using Spark Dataframe APIs -DLlib supports Spark Dataframes as the input to the distributed training, and as -the input/output of the distributed inference. Consequently, the user can easily -process large-scale dataset using Apache Spark, and directly apply AI models on -the distributed (and possibly in-memory) Dataframes without data conversion or serialization - -We create Spark session so we can use Spark API to load and process the data -``` -val spark = new SQLContext(sc) -``` - -1. We can use Spark API to load the data into Spark DataFrame, eg. read csv file into Spark DataFrame -``` -val path = "pima-indians-diabetes.data.csv" -val df = spark.read.options(Map("inferSchema"->"true","delimiter"->",")).csv(path) - .toDF("num_times_pregrant", "plasma_glucose", "blood_pressure", "skin_fold_thickness", "2-hour_insulin", "body_mass_index", "diabetes_pedigree_function", "age", "class") -``` - -If the feature column for the model is a Spark ML Vector. Please assemble related columns into a Vector and pass it to the model. eg. -``` -val assembler = new VectorAssembler() - .setInputCols(Array("num_times_pregrant", "plasma_glucose", "blood_pressure", "skin_fold_thickness", "2-hour_insulin", "body_mass_index", "diabetes_pedigree_function", "age")) - .setOutputCol("features") -val assembleredDF = assembler.transform(df) -val df2 = assembleredDF.withColumn("label",col("class").cast(DoubleType) + lit(1)) -``` - -2. If the training data is image, we can use DLLib api to load image into Spark DataFrame. Eg. -``` -val createLabel = udf { row: Row => -if (new Path(row.getString(0)).getName.contains("cat")) 1 else 2 -} -val imagePath = "cats_dogs/" -val imgDF = NNImageReader.readImages(imagePath, sc) -``` - -It will load the images and generate feature tensors automatically. Also we need generate labels ourselves. eg: -``` -val df = imgDF.withColumn("label", createLabel(col("image"))) -``` - -Then split the Spark DataFrame into traing part and validation part -``` -val Array(trainDF, valDF) = df.randomSplit(Array(0.8, 0.2)) -``` - -## 4. Model Definition - -#### Using Keras-like APIs - -To define a model, you can use the [Keras Style API](https://bigdl.readthedocs.io/en/latest/doc/DLlib/Overview/keras-api.html). -``` -val x1 = Input(Shape(8)) -val dense1 = Dense(12, activation="relu").inputs(x1) -val dense2 = Dense(8, activation="relu").inputs(dense1) -val dense3 = Dense(2).inputs(dense2) -val dmodel = Model(x1, dense3) -``` - -After creating the model, you will have to decide which loss function to use in training. - -Now you can use `compile` function of the model to set the loss function, optimization method. -``` -dmodel.compile(optimizer = new Adam(), loss = ClassNLLCriterion()) -``` - -Now the model is built and ready to train. - -## 5. Distributed Model Training -Now you can use 'fit' begin the training, please set the label columns. Model Evaluation can be performed periodically during a training. -1. If the dataframe is generated using Spark apis, you also need set the feature columns. eg. -``` -model.fit(x=trainDF, batchSize=4, nbEpoch = 2, - featureCols = Array("feature1"), labelCols = Array("label"), valX=valDF) -``` -Note: Above model accepts single input(column `feature1`) and single output(column `label`). - -If your model accepts multiple inputs(eg. column `f1`, `f2`, `f3`), please set the features as below: -``` -model.fit(x=dataframe, batchSize=4, nbEpoch = 2, - featureCols = Array("f1", "f2", "f3"), labelCols = Array("label")) -``` - -Similarly, if the model accepts multiple outputs(eg. column `label1`, `label2`), please set the label columns as below: -``` -model.fit(x=dataframe, batchSize=4, nbEpoch = 2, - featureCols = Array("f1", "f2", "f3"), labelCols = Array("label1", "label2")) -``` - -2. If the dataframe is generated using DLLib `NNImageReader`, we don't need set `featureCols`, we can set `transform` to config how to process the images before training. Eg. -``` -val transformers = transforms.Compose(Array(ImageResize(50, 50), - ImageMirror())) -model.fit(x=dataframe, batchSize=4, nbEpoch = 2, - labelCols = Array("label"), transform = transformers) -``` -For more details about how to use DLLib keras api to train image data, you may want to refer [ImageClassification](https://github.com/intel-analytics/BigDL/blob/main/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras/ImageClassification.scala) - -## 6. Model saving and loading -When training is finished, you may need to save the final model for later use. - -BigDL allows you to save your BigDL model on local filesystem, HDFS, or Amazon s3. -- **save** -``` -val modelPath = "/tmp/demo/keras.model" -dmodel.saveModel(modelPath) -``` - -- **load** -``` -val loadModel = Models.loadModel(modelPath) - -val preDF2 = loadModel.predict(valDF, featureCols = Array("features"), predictionCol = "predict") -``` - -You may want to refer [Save/Load](https://bigdl.readthedocs.io/en/latest/doc/DLlib/Overview/keras-api.html#save) - -## 7. Distributed evaluation and inference -After training finishes, you can then use the trained model for prediction or evaluation. - -- **inference** -1. For dataframe generated by Spark API, please set `featureCols` -``` -dmodel.predict(trainDF, featureCols = Array("features"), predictionCol = "predict") -``` -2. For dataframe generated by `NNImageReader`, no need to set `featureCols` and you can set `transform` if needed -``` -model.predict(imgDF, predictionCol = "predict", transform = transformers) -``` - -- **evaluation** -Similary for dataframe generated by Spark API, the code is as below: -``` -dmodel.evaluate(trainDF, batchSize = 4, featureCols = Array("features"), - labelCols = Array("label")) -``` - -For dataframe generated by `NNImageReader`: -``` -model.evaluate(imgDF, batchSize = 1, labelCols = Array("label"), transform = transformers) -``` - -## 8. Checkpointing and resuming training -You can configure periodically taking snapshots of the model. -``` -val cpPath = "/tmp/demo/cp" -dmodel.setCheckpoint(cpPath, overWrite=false) -``` -You can also set ```overWrite``` to ```true``` to enable overwriting any existing snapshot files - -After training stops, you can resume from any saved point. Choose one of the model snapshots to resume (saved in checkpoint path, details see Checkpointing). Use Models.loadModel to load the model snapshot into an model object. -``` -val loadModel = Models.loadModel(path) -``` - -## 9. Monitor your training - -- **Tensorboard** - -BigDL provides a convenient way to monitor/visualize your training progress. It writes the statistics collected during training/validation. Saved summary can be viewed via TensorBoard. - -In order to take effect, it needs to be called before fit. -``` -dmodel.setTensorBoard("./", "dllib_demo") -``` -For more details, please refer [visulization](visualization.md) - -## 10. Transfer learning and finetuning - -- **freeze and trainable** -BigDL DLLib supports exclude some layers of model from training. -``` -dmodel.freeze(layer_names) -``` -Layers that match the given names will be freezed. If a layer is freezed, its parameters(weight/bias, if exists) are not changed in training process. - -BigDL DLLib also support unFreeze operations. The parameters for the layers that match the given names will be trained(updated) in training process -``` -dmodel.unFreeze(layer_names) -``` -For more information, you may refer [freeze](freeze.md) - -## 11. Hyperparameter tuning -- **optimizer** - -DLLib supports a list of optimization methods. -For more details, please refer [optimization](optim-Methods.md) - -- **learning rate scheduler** - -DLLib supports a list of learning rate scheduler. -For more details, please refer [lr_scheduler](learningrate-Scheduler.md) - -- **batch size** - -DLLib supports set batch size during training and prediction. We can adjust the batch size to tune the model's accuracy. - -- **regularizer** - -DLLib supports a list of regularizers. -For more details, please refer [regularizer](regularizers.md) - -- **clipping** - -DLLib supports gradient clipping operations. -For more details, please refer [gradient_clip](clipping.md) - -## 12. Running program -You can run a bigdl-dllib program as a standard Spark program (running on either a local machine or a distributed cluster) as follows: -``` -# Spark local mode -${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ - --master local[2] \ - --class class_name \ - jar_path - -# Spark standalone mode -## ${SPARK_HOME}/sbin/start-master.sh -## check master URL from http://localhost:8080 -${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ - --master spark://... \ - --executor-cores cores_per_executor \ - --total-executor-cores total_cores_for_the_job \ - --class class_name \ - jar_path - -# Spark yarn client mode -${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ - --master yarn \ - --deploy-mode client \ - --executor-cores cores_per_executor \ - --num-executors executors_number \ - --class class_name \ - jar_path - -# Spark yarn cluster mode -${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ - --master yarn \ - --deploy-mode cluster \ - --executor-cores cores_per_executor \ - --num-executors executors_number \ - --class class_name - jar_path -``` -For more detail about how to run BigDL scala application, please refer https://bigdl.readthedocs.io/en/latest/doc/UserGuide/scala.html diff --git a/docs/readthedocs/source/doc/DLlib/Overview/visualization.md b/docs/readthedocs/source/doc/DLlib/Overview/visualization.md index 23006116..00e8ed0c 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/visualization.md +++ b/docs/readthedocs/source/doc/DLlib/Overview/visualization.md @@ -1,5 +1,5 @@ ## **Visualizing training with TensorBoard** -With the summary info generated, we can then use [TensorBoard](https://pypi.python.org/pypi/tensorboard) to visualize the behaviors of the BigDL program. +With the summary info generated, we can then use [TensorBoard](https://pypi.python.org/pypi/tensorboard) to visualize the behaviors of the BigDL program. * **Installing TensorBoard** @@ -31,10 +31,10 @@ After that, navigate to the TensorBoard dashboard using a browser. You can find * **Visualizations in TensorBoard** Within the TensorBoard dashboard, you will be able to read the visualizations of each run, including the “Loss” and “Throughput” curves under the SCALARS tab (as illustrated below): -![Scalar](../Image/tensorboard-scalar.png) +![](../Image/tensorboard-scalar.png) And “weights”, “bias”, “gradientWeights” and “gradientBias” under the DISTRIBUTIONS and HISTOGRAMS tabs (as illustrated below): -![histogram1](../Image/tensorboard-histo1.png) -![histogram2](../Image/tensorboard-histo2.png) +![](../Image/tensorboard-histo1.png) +![](../Image/tensorboard-histo2.png) --- \ No newline at end of file diff --git a/docs/readthedocs/source/doc/DLlib/QuickStart/index.md b/docs/readthedocs/source/doc/DLlib/QuickStart/index.md new file mode 100644 index 00000000..06a16e3a --- /dev/null +++ b/docs/readthedocs/source/doc/DLlib/QuickStart/index.md @@ -0,0 +1,9 @@ +# DLlib Tutorial + + +- [**Python Quickstart Notebook**](./python-getting-started.html) + + > ![](../../../../image/colab_logo_32px.png)[Run in Google Colab](https://colab.research.google.com/github/intel-analytics/BigDL/blob/branch-2.0/python/dllib/colab-notebook/dllib_keras_api.ipynb)  ![](../../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/blob/branch-2.0/python/dllib/colab-notebook/dllib_keras_api.ipynb) + + In this guide we will demonstrate how to use _DLlib keras style api_ and _DLlib NNClassifier_ for classification. + diff --git a/docs/readthedocs/source/doc/DLlib/QuickStart/python-getting-started.md b/docs/readthedocs/source/doc/DLlib/QuickStart/python-getting-started.md new file mode 100644 index 00000000..b6f6c736 --- /dev/null +++ b/docs/readthedocs/source/doc/DLlib/QuickStart/python-getting-started.md @@ -0,0 +1,218 @@ +# DLLib Python Getting Start Guide + +## 1. Code initialization +```nncontext``` is the main entry for provisioning the dllib program on the underlying cluster (such as K8s or Hadoop cluster), or just on a single laptop. + +It is recommended to initialize `nncontext` at the beginning of your program: +``` +from bigdl.dllib.nncontext import * +sc = init_nncontext() +``` +For more information about ```nncontext```, please refer to [nncontext](../Overview/dllib.md#initialize-nn-context) + +## 2. Distributed Data Loading + +#### Using Spark Dataframe APIs +DLlib supports Spark Dataframes as the input to the distributed training, and as +the input/output of the distributed inference. Consequently, the user can easily +process large-scale dataset using Apache Spark, and directly apply AI models on +the distributed (and possibly in-memory) Dataframes without data conversion or serialization + +We create Spark session so we can use Spark API to load and process the data +``` +spark = SQLContext(sc) +``` + +1. We can use Spark API to load the data into Spark DataFrame, eg. read csv file into Spark DataFrame + ``` + path = "pima-indians-diabetes.data.csv" + spark.read.csv(path) + ``` + + If the feature column for the model is a Spark ML Vector. Please assemble related columns into a Vector and pass it to the model. eg. + ``` + from pyspark.ml.feature import VectorAssembler + vecAssembler = VectorAssembler(outputCol="features") + vecAssembler.setInputCols(["num_times_pregrant", "plasma_glucose", "blood_pressure", "skin_fold_thickness", "2-hour_insulin", "body_mass_index", "diabetes_pedigree_function", "age"]) + assemble_df = vecAssembler.transform(df) + assemble_df.withColumn("label", col("class").cast(DoubleType) + lit(1)) + ``` + +2. If the training data is image, we can use DLLib api to load image into Spark DataFrame. Eg. + ``` + imgPath = "cats_dogs/" + imageDF = NNImageReader.readImages(imgPath, sc) + ``` + + It will load the images and generate feature tensors automatically. Also we need generate labels ourselves. eg: + ``` + labelDF = imageDF.withColumn("name", getName(col("image"))) \ + .withColumn("label", getLabel(col('name'))) + ``` + + Then split the Spark DataFrame into traing part and validation part + ``` + (trainingDF, validationDF) = labelDF.randomSplit([0.9, 0.1]) + ``` + +## 3. Model Definition + +#### Using Keras-like APIs + +To define a model, you can use the [Keras Style API](../Overview/keras-api.md). +``` +x1 = Input(shape=[8]) +dense1 = Dense(12, activation="relu")(x1) +dense2 = Dense(8, activation="relu")(dense1) +dense3 = Dense(2)(dense2) +dmodel = Model(input=x1, output=dense3) +``` + +After creating the model, you will have to decide which loss function to use in training. + +Now you can use `compile` function of the model to set the loss function, optimization method. +``` +dmodel.compile(optimizer = "adam", loss = "sparse_categorical_crossentropy") +``` + +Now the model is built and ready to train. + +## 4. Distributed Model Training +Now you can use 'fit' begin the training, please set the label columns. Model Evaluation can be performed periodically during a training. +1. If the dataframe is generated using Spark apis, you also need set the feature columns. eg. + ``` + model.fit(df, feature_cols=["features"], label_cols=["label"], batch_size=4, nb_epoch=1) + ``` + Note: Above model accepts single input(column `features`) and single output(column `label`). + + If your model accepts multiple inputs(eg. column `f1`, `f2`, `f3`), please set the features as below: + ``` + model.fit(df, feature_cols=["f1", "f2"], label_cols=["label"], batch_size=4, nb_epoch=1) + ``` + + Similarly, if the model accepts multiple outputs(eg. column `label1`, `label2`), please set the label columns as below: + ``` + model.fit(df, feature_cols=["features"], label_cols=["l1", "l2"], batch_size=4, nb_epoch=1) + ``` + +2. If the dataframe is generated using DLLib `NNImageReader`, we don't need set `feature_cols`, we can set `transform` to config how to process the images before training. Eg. + ``` + from bigdl.dllib.feature.image import transforms + transformers = transforms.Compose([ImageResize(50, 50), ImageMirror()]) + model.fit(image_df, label_cols=["label"], batch_size=1, nb_epoch=1, transform=transformers) + ``` + For more details about how to use DLLib keras api to train image data, you may want to refer [ImageClassification](https://github.com/intel-analytics/BigDL/tree/main/python/dllib/examples/keras/image_classification.py) + +## 5. Model saving and loading +When training is finished, you may need to save the final model for later use. + +BigDL allows you to save your BigDL model on local filesystem, HDFS, or Amazon s3. +- **save** + ``` + modelPath = "/tmp/demo/keras.model" + dmodel.saveModel(modelPath) + ``` + +- **load** + ``` + loadModel = Model.loadModel(modelPath) + preDF = loadModel.predict(df, feature_cols=["features"], prediction_col="predict") + ``` + +You may want to refer [Save/Load](../Overview/keras-api.html#save) + +## 6. Distributed evaluation and inference +After training finishes, you can then use the trained model for prediction or evaluation. + +- **inference** + 1. For dataframe generated by Spark API, please set `feature_cols` and `prediction_col` + ``` + dmodel.predict(df, feature_cols=["features"], prediction_col="predict") + ``` + 2. For dataframe generated by `NNImageReader`, please set `prediction_col` and you can set `transform` if needed + ``` + model.predict(df, prediction_col="predict", transform=transformers) + ``` + +- **evaluation** + + Similary for dataframe generated by Spark API, the code is as below: + ``` + dmodel.evaluate(df, batch_size=4, feature_cols=["features"], label_cols=["label"]) + ``` + + For dataframe generated by `NNImageReader`: + ``` + model.evaluate(image_df, batch_size=1, label_cols=["label"], transform=transformers) + ``` + +## 7. Checkpointing and resuming training +You can configure periodically taking snapshots of the model. +``` +cpPath = "/tmp/demo/cp" +dmodel.set_checkpoint(cpPath) +``` +You can also set ```over_write``` to ```true``` to enable overwriting any existing snapshot files + +After training stops, you can resume from any saved point. Choose one of the model snapshots to resume (saved in checkpoint path, details see Checkpointing). Use Models.loadModel to load the model snapshot into an model object. +``` +loadModel = Model.loadModel(path) +``` + +## 8. Monitor your training + +- **Tensorboard** + + BigDL provides a convenient way to monitor/visualize your training progress. It writes the statistics collected during training/validation. Saved summary can be viewed via TensorBoard. + + In order to take effect, it needs to be called before fit. + ``` + dmodel.set_tensorboard("./", "dllib_demo") + ``` + For more details, please refer [visulization](../Overview/visualization.md) + +## 9. Transfer learning and finetuning + +- **freeze and trainable** + + BigDL DLLib supports exclude some layers of model from training. + ``` + dmodel.freeze(layer_names) + ``` + Layers that match the given names will be freezed. If a layer is freezed, its parameters(weight/bias, if exists) are not changed in training process. + + BigDL DLLib also support unFreeze operations. The parameters for the layers that match the given names will be trained(updated) in training process + ``` + dmodel.unFreeze(layer_names) + ``` + For more information, you may refer [freeze](../../PythonAPI/DLlib/freeze.md) + +## 10. Hyperparameter tuning +- **optimizer** + + DLLib supports a list of optimization methods. + For more details, please refer [optimization](../../PythonAPI/DLlib/optim-Methods.md) + +- **learning rate scheduler** + + DLLib supports a list of learning rate scheduler. + For more details, please refer [lr_scheduler](../../PythonAPI/DLlib/learningrate-Scheduler.md) + +- **batch size** + + DLLib supports set batch size during training and prediction. We can adjust the batch size to tune the model's accuracy. + +- **regularizer** + + DLLib supports a list of regularizers. + For more details, please refer [regularizer](../../PythonAPI/DLlib/regularizers.md) + +- **clipping** + + DLLib supports gradient clipping operations. + For more details, please refer [gradient_clip](../../PythonAPI/DLlib/clipping.md) + +## 11. Running program +``` +python you_app_code.py +``` diff --git a/docs/readthedocs/source/doc/DLlib/QuickStart/scala-getting-started.md b/docs/readthedocs/source/doc/DLlib/QuickStart/scala-getting-started.md new file mode 100644 index 00000000..c08bc722 --- /dev/null +++ b/docs/readthedocs/source/doc/DLlib/QuickStart/scala-getting-started.md @@ -0,0 +1,303 @@ +# DLLib Scala Getting Start Guide + +## 1. Creating dev environment + +#### Scala project (maven & sbt) + +- **Maven** + + To use BigDL DLLib to build your own deep learning application, you can use maven to create your project and add bigdl-dllib to your dependency. Please add below code to your pom.xml to add BigDL DLLib as your dependency: + ``` + + com.intel.analytics.bigdl + bigdl-dllib-spark_2.4.6 + 0.14.0 + + ``` + +- **SBT** + ``` + libraryDependencies += "com.intel.analytics.bigdl" % "bigdl-dllib-spark_2.4.6" % "0.14.0" + ``` + For more information about how to add BigDL dependency, please refer [scala docs](../../UserGuide/scala.md#build-a-scala-project) + +#### IDE (Intelij) +Open up IntelliJ and click File => Open + +Navigate to your project. If you have add BigDL DLLib as dependency in your pom.xml. +The IDE will automatically download it from maven and you are able to run your application. + +For more details about how to setup IDE for BigDL project, please refer [IDE Setup Guide](../../UserGuide/develop.html#id2) + + +## 2. Code initialization +```NNContext``` is the main entry for provisioning the dllib program on the underlying cluster (such as K8s or Hadoop cluster), or just on a single laptop. + +It is recommended to initialize `NNContext` at the beginning of your program: +``` +import com.intel.analytics.bigdl.dllib.NNContext +import com.intel.analytics.bigdl.dllib.keras.Model +import com.intel.analytics.bigdl.dllib.keras.models.Models +import com.intel.analytics.bigdl.dllib.keras.optimizers.Adam +import com.intel.analytics.bigdl.dllib.nn.ClassNLLCriterion +import com.intel.analytics.bigdl.dllib.utils.Shape +import com.intel.analytics.bigdl.dllib.keras.layers._ +import com.intel.analytics.bigdl.numeric.NumericFloat +import org.apache.spark.ml.feature.VectorAssembler +import org.apache.spark.sql.SQLContext +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.DoubleType + +val sc = NNContext.initNNContext("dllib_demo") +``` +For more information about ```NNContext```, please refer to [NNContext](../Overview/dllib.md#initialize-nn-context) + +## 3. Distributed Data Loading + +#### Using Spark Dataframe APIs +DLlib supports Spark Dataframes as the input to the distributed training, and as +the input/output of the distributed inference. Consequently, the user can easily +process large-scale dataset using Apache Spark, and directly apply AI models on +the distributed (and possibly in-memory) Dataframes without data conversion or serialization + +We create Spark session so we can use Spark API to load and process the data +``` +val spark = new SQLContext(sc) +``` + +1. We can use Spark API to load the data into Spark DataFrame, eg. read csv file into Spark DataFrame + ``` + val path = "pima-indians-diabetes.data.csv" + val df = spark.read.options(Map("inferSchema"->"true","delimiter"->",")).csv(path) + .toDF("num_times_pregrant", "plasma_glucose", "blood_pressure", "skin_fold_thickness", "2-hour_insulin", "body_mass_index", "diabetes_pedigree_function", "age", "class") + ``` + + If the feature column for the model is a Spark ML Vector. Please assemble related columns into a Vector and pass it to the model. eg. + ``` + val assembler = new VectorAssembler() + .setInputCols(Array("num_times_pregrant", "plasma_glucose", "blood_pressure", "skin_fold_thickness", "2-hour_insulin", "body_mass_index", "diabetes_pedigree_function", "age")) + .setOutputCol("features") + val assembleredDF = assembler.transform(df) + val df2 = assembleredDF.withColumn("label",col("class").cast(DoubleType) + lit(1)) + ``` + +2. If the training data is image, we can use DLLib api to load image into Spark DataFrame. Eg. + ``` + val createLabel = udf { row: Row => + if (new Path(row.getString(0)).getName.contains("cat")) 1 else 2 + } + val imagePath = "cats_dogs/" + val imgDF = NNImageReader.readImages(imagePath, sc) + ``` + + It will load the images and generate feature tensors automatically. Also we need generate labels ourselves. eg: + ``` + val df = imgDF.withColumn("label", createLabel(col("image"))) + ``` + + Then split the Spark DataFrame into traing part and validation part + ``` + val Array(trainDF, valDF) = df.randomSplit(Array(0.8, 0.2)) + ``` + +## 4. Model Definition + +#### Using Keras-like APIs + +To define a model, you can use the [Keras Style API](../Overview/keras-api.md). +``` +val x1 = Input(Shape(8)) +val dense1 = Dense(12, activation="relu").inputs(x1) +val dense2 = Dense(8, activation="relu").inputs(dense1) +val dense3 = Dense(2).inputs(dense2) +val dmodel = Model(x1, dense3) +``` + +After creating the model, you will have to decide which loss function to use in training. + +Now you can use `compile` function of the model to set the loss function, optimization method. +``` +dmodel.compile(optimizer = new Adam(), loss = ClassNLLCriterion()) +``` + +Now the model is built and ready to train. + +## 5. Distributed Model Training +Now you can use 'fit' begin the training, please set the label columns. Model Evaluation can be performed periodically during a training. +1. If the dataframe is generated using Spark apis, you also need set the feature columns. eg. + ``` + model.fit(x=trainDF, batchSize=4, nbEpoch = 2, + featureCols = Array("feature1"), labelCols = Array("label"), valX=valDF) + ``` + Note: Above model accepts single input(column `feature1`) and single output(column `label`). + + If your model accepts multiple inputs(eg. column `f1`, `f2`, `f3`), please set the features as below: + ``` + model.fit(x=dataframe, batchSize=4, nbEpoch = 2, + featureCols = Array("f1", "f2", "f3"), labelCols = Array("label")) + ``` + + Similarly, if the model accepts multiple outputs(eg. column `label1`, `label2`), please set the label columns as below: + ``` + model.fit(x=dataframe, batchSize=4, nbEpoch = 2, + featureCols = Array("f1", "f2", "f3"), labelCols = Array("label1", "label2")) + ``` + +2. If the dataframe is generated using DLLib `NNImageReader`, we don't need set `featureCols`, we can set `transform` to config how to process the images before training. Eg. + ``` + val transformers = transforms.Compose(Array(ImageResize(50, 50), + ImageMirror())) + model.fit(x=dataframe, batchSize=4, nbEpoch = 2, + labelCols = Array("label"), transform = transformers) + ``` + For more details about how to use DLLib keras api to train image data, you may want to refer [ImageClassification](https://github.com/intel-analytics/BigDL/blob/main/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/example/keras/ImageClassification.scala) + +## 6. Model saving and loading +When training is finished, you may need to save the final model for later use. + +BigDL allows you to save your BigDL model on local filesystem, HDFS, or Amazon s3. +- **save** + ``` + val modelPath = "/tmp/demo/keras.model" + dmodel.saveModel(modelPath) + ``` + +- **load** + ``` + val loadModel = Models.loadModel(modelPath) + + val preDF2 = loadModel.predict(valDF, featureCols = Array("features"), predictionCol = "predict") + ``` + +You may want to refer [Save/Load](../Overview/keras-api.html#save) + +## 7. Distributed evaluation and inference +After training finishes, you can then use the trained model for prediction or evaluation. + +- **inference** + 1. For dataframe generated by Spark API, please set `featureCols` + ``` + dmodel.predict(trainDF, featureCols = Array("features"), predictionCol = "predict") + ``` + 2. For dataframe generated by `NNImageReader`, no need to set `featureCols` and you can set `transform` if needed + ``` + model.predict(imgDF, predictionCol = "predict", transform = transformers) + ``` + +- **evaluation** + + Similary for dataframe generated by Spark API, the code is as below: + ``` + dmodel.evaluate(trainDF, batchSize = 4, featureCols = Array("features"), + labelCols = Array("label")) + ``` + + For dataframe generated by `NNImageReader`: + ``` + model.evaluate(imgDF, batchSize = 1, labelCols = Array("label"), transform = transformers) + ``` + +## 8. Checkpointing and resuming training +You can configure periodically taking snapshots of the model. +``` +val cpPath = "/tmp/demo/cp" +dmodel.setCheckpoint(cpPath, overWrite=false) +``` +You can also set ```overWrite``` to ```true``` to enable overwriting any existing snapshot files + +After training stops, you can resume from any saved point. Choose one of the model snapshots to resume (saved in checkpoint path, details see Checkpointing). Use Models.loadModel to load the model snapshot into an model object. +``` +val loadModel = Models.loadModel(path) +``` + +## 9. Monitor your training + +- **Tensorboard** + + BigDL provides a convenient way to monitor/visualize your training progress. It writes the statistics collected during training/validation. Saved summary can be viewed via TensorBoard. + + In order to take effect, it needs to be called before fit. + ``` + dmodel.setTensorBoard("./", "dllib_demo") + ``` + For more details, please refer [visulization](../Overview/visualization.md)` + +## 10. Transfer learning and finetuning + +- **freeze and trainable** + + BigDL DLLib supports exclude some layers of model from training. + ``` + dmodel.freeze(layer_names) + ``` + Layers that match the given names will be freezed. If a layer is freezed, its parameters(weight/bias, if exists) are not changed in training process. + + BigDL DLLib also support unFreeze operations. The parameters for the layers that match the given names will be trained(updated) in training process + ``` + dmodel.unFreeze(layer_names) + ``` + For more information, you may refer [freeze](../../PythonAPI/DLlib/freeze.md) + +## 11. Hyperparameter tuning +- **optimizer** + + DLLib supports a list of optimization methods. + For more details, please refer [optimization](../../PythonAPI/DLlib/optim-Methods.md) + +- **learning rate scheduler** + + DLLib supports a list of learning rate scheduler. + For more details, please refer [lr_scheduler](../../PythonAPI/DLlib/learningrate-Scheduler.md) + +- **batch size** + + DLLib supports set batch size during training and prediction. We can adjust the batch size to tune the model's accuracy. + +- **regularizer** + + DLLib supports a list of regularizers. + For more details, please refer [regularizer](../../PythonAPI/DLlib/regularizers.md) + +- **clipping** + + DLLib supports gradient clipping operations. + For more details, please refer [gradient_clip](../../PythonAPI/DLlib/clipping.md) + +## 12. Running program +You can run a bigdl-dllib program as a standard Spark program (running on either a local machine or a distributed cluster) as follows: +``` +# Spark local mode +${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ + --master local[2] \ + --class class_name \ + jar_path + +# Spark standalone mode +## ${SPARK_HOME}/sbin/start-master.sh +## check master URL from http://localhost:8080 +${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ + --master spark://... \ + --executor-cores cores_per_executor \ + --total-executor-cores total_cores_for_the_job \ + --class class_name \ + jar_path + +# Spark yarn client mode +${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ + --master yarn \ + --deploy-mode client \ + --executor-cores cores_per_executor \ + --num-executors executors_number \ + --class class_name \ + jar_path + +# Spark yarn cluster mode +${BIGDL_HOME}/bin/spark-submit-with-dllib.sh \ + --master yarn \ + --deploy-mode cluster \ + --executor-cores cores_per_executor \ + --num-executors executors_number \ + --class class_name + jar_path +``` +For more detail about how to run BigDL scala application, please refer to [Scala UserGuide](../../UserGuide/scala.md) diff --git a/docs/readthedocs/source/doc/DLlib/index.rst b/docs/readthedocs/source/doc/DLlib/index.rst new file mode 100644 index 00000000..e6876eaf --- /dev/null +++ b/docs/readthedocs/source/doc/DLlib/index.rst @@ -0,0 +1,62 @@ +BigDL-DLlib +========================= + +**BigDL-DLlib** (or **DLlib** for short) is a distributed deep learning library for Apache Spark; with DLlib, users can write their deep learning applications as standard Spark programs (using either Scala or Python APIs). + +------- + + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + Documents in these sections helps you getting started quickly with DLLib. + + +++ + :bdg-link:`DLlib in 5 minutes <./Overview/dllib.html>` | + :bdg-link:`Installation <./Overview/install.html>` + + .. grid-item-card:: + + **Key Features Guide** + ^^^ + + Each guide in this section provides you with in-depth information, concepts and knowledges about DLLib key features. + + +++ + + :bdg-link:`Keras-Like API <./Overview/keras-api.html>` | + :bdg-link:`Spark ML Pipeline <./Overview/nnframes.html>` + + .. grid-item-card:: + + **Examples** + ^^^ + + DLLib Examples and Tutorials. + + +++ + + :bdg-link:`Tutorials <./QuickStart/index.html>` + + .. grid-item-card:: + + **API Document** + ^^^ + + API Document provides detailed description of DLLib APIs. + + +++ + + :bdg-link:`API Document <../PythonAPI/DLlib/index.html>` + + +.. toctree:: + :hidden: + + BigDL-DLlib Document + diff --git a/docs/readthedocs/source/doc/Friesian/examples.md b/docs/readthedocs/source/doc/Friesian/examples.md new file mode 100644 index 00000000..111db0e9 --- /dev/null +++ b/docs/readthedocs/source/doc/Friesian/examples.md @@ -0,0 +1,70 @@ +### Use Cases + + +- **Train a DeepFM model using recsys data** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/deep_fm) + +--------------------------- + +- **Run DeepRec with BigDL** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/deeprec) + +--------------------------- + +- **Train DIEN using the Amazon Book Reviews dataset** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/dien) + +--------------------------- + +- **Preprocess the Criteo dataset for DLRM Model** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/dlrm) + +--------------------------- + +- **Train an LightGBM model using Twitter dataset** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/lightGBM) + +--------------------------- + +- **Running Friesian listwise example** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/listwise_ranking) + +--------------------------- + +- **Multi-task Recommendation with BigDL** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/multi_task) + +--------------------------- + +- **Train an NCF model on MovieLens** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/ncf) + + +--------------------------- + +- **Offline Recall with Faiss on Spark** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/recall) + + +--------------------------- + +- **Recommend items using Friesian-Serving Framework** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/serving) + + +--------------------------- + +- **Train a two tower model using recsys data** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/two_tower) + +--------------------------- + +- **Preprocess the Criteo dataset for WideAndDeep Model** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/wnd) + + +--------------------------- + +- **Train an XGBoost model using Twitter dataset** +>![](../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/tree/main/python/friesian/example/xgb) + diff --git a/docs/readthedocs/source/doc/Friesian/index.rst b/docs/readthedocs/source/doc/Friesian/index.rst new file mode 100644 index 00000000..16676873 --- /dev/null +++ b/docs/readthedocs/source/doc/Friesian/index.rst @@ -0,0 +1,66 @@ +BigDL-Friesian +========================= + + + +BigDL Friesian is an application framework for building optimized large-scale recommender solutions. The recommending workflows built on top of Friesian can seamlessly scale out to distributed big data clusters in the production environment. + +Friesian provides end-to-end support for three typical stages in a modern recommendation system: + +- Offline stage: distributed feature engineering and model training. +- Nearline stage: Feature and model updates. +- Online stage: Recall and ranking. + +------- + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + Documents in these sections helps you getting started quickly with Friesian. + + +++ + + :bdg-link:`Introduction <./intro.html>` + + .. grid-item-card:: + + **Key Features Guide** + ^^^ + + Each guide in this section provides you with in-depth information, concepts and knowledges about Friesian key features. + + +++ + + :bdg-link:`Serving <./serving.html>` + + .. grid-item-card:: + + **Use Cases** + ^^^ + + Use Cases and Examples. + + +++ + + :bdg-link:`Use Cases <./examples.html>` + + .. grid-item-card:: + + **API Document** + ^^^ + + API Document provides detailed description of Nano APIs. + + +++ + + :bdg-link:`API Document <../PythonAPI/Friesian/index.html>` + +.. toctree:: + :hidden: + + BigDL-Friesian Document \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Friesian/intro.rst b/docs/readthedocs/source/doc/Friesian/intro.rst new file mode 100644 index 00000000..5468c0e9 --- /dev/null +++ b/docs/readthedocs/source/doc/Friesian/intro.rst @@ -0,0 +1,17 @@ +Friesian Introduction +========================== + +BigDL Friesian is an application framework for building optimized large-scale recommender solutions. The recommending workflows built on top of Friesian can seamlessly scale out to distributed big data clusters in the production environment. + +Friesian provides end-to-end support for three typical stages in a modern recommendation system: + +- Offline stage: distributed feature engineering and model training. +- Nearline stage: Feature and model updates. +- Online stage: Recall and ranking. + +The overall architecture of Friesian is shown in the following diagram: + + +.. image:: ../../../image/friesian_architecture.png + + diff --git a/docs/readthedocs/source/doc/Friesian/serving.md b/docs/readthedocs/source/doc/Friesian/serving.md new file mode 100644 index 00000000..405c8b94 --- /dev/null +++ b/docs/readthedocs/source/doc/Friesian/serving.md @@ -0,0 +1,600 @@ +## Serving Recommendation Framework + +### Architecture of the serving pipelines + +The diagram below demonstrates the components of the friesian serving system, which typically consists of three stages: + +- Offline: Preprocess the data to get user/item DNN features and user/item Embedding features. Then use the embedding features and embedding model to get embedding vectors. +- Nearline: Retrieve user/item profiles and keep them in the Key-Value store. Retrieve item embedding vectors and build the faiss index. Make updates to the profiles from time to time. +- Online: Trigger the recommendation process whenever a user comes. Recall service generate candidates from millions of items based on embeddings and the deep learning model ranks the candidates for the final recommendation results. + +![](../../../image/friesian_architecture.png) + + +### Services and APIs +The friesian serving system consists of 4 types of services: +- Ranking Service: performs model inference and returns the results. + - `rpc doPredict(Content) returns (Prediction) {}` + - Input: The `encodeStr` is a Base64 string encoded from a bigdl [Activity](https://github.com/intel-analytics/BigDL/blob/branch-2.0/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/Activity.scala) serialized byte array. + ```bash + message Content { + string encodedStr = 1; + } + ``` + - Output: The `predictStr` is a Base64 string encoded from a bigdl [Activity](https://github.com/intel-analytics/BigDL/blob/branch-2.0/scala/dllib/src/main/scala/com/intel/analytics/bigdl/dllib/nn/abstractnn/Activity.scala) (the inference result) serialized byte array. + ```bash + message Prediction { + string predictStr = 1; + } + ``` +- Feature Service: searches user embeddings, user features or item features in Redis, and returns the features. + - `rpc getUserFeatures(IDs) returns (Features) {}` and `rpc getItemFeatures(IDs) returns (Features) {}` + - Input: The user/item id list for searching. + ```bash + message IDs { + repeated int32 ID = 1; + } + ``` + - Output: `colNames` is a string list of the column names. `b64Feature` is a list of Base64 string, each string is encoded from java serialized array of objects. `ID` is a list of ids corresponding `b64Feature`. + ```bash + message Features { + repeated string colNames = 1; + repeated string b64Feature = 2; + repeated int32 ID = 3; + } + ``` +- Recall Service: searches item candidates in the built faiss index and returns candidates id list. + - `rpc searchCandidates(Query) returns (Candidates) {}` + - Input: `userID` is the id of the user to search similar item candidates. `k` is the number of candidates. + ```bash + message Query { + int32 userID = 1; + int32 k = 2; + } + ``` + - Output: `candidate` is the list of ids of item candidates. + ```bash + message Candidates { + repeated int32 candidate = 1; + } + ``` +- Recommender Service: gets candidates from the recall service, calls the feature service to get the user and item candidate's features, then sorts the inference results from ranking service and returns the top recommendNum items. + - `rpc getRecommendIDs(RecommendRequest) returns (RecommendIDProbs) {}` + - Input: `ID` is a list of user ids to recommend. `recommendNum` is the number of items to recommend. `candidateNum` is the number of generated candidates to inference in ranking service. + ```bash + message RecommendRequest { + int32 recommendNum = 1; + int32 candidateNum = 2; + repeated int32 ID = 3; + } + ``` + - Output: `IDProbList` is a list of results corresponding to user `ID` in input. Each `IDProbs` consists of `ID` and `prob`, `ID` is the list of item ids, and `prob` is the corresponding probability. + ```bash + message RecommendIDProbs { + repeated IDProbs IDProbList = 1; + } + message IDProbs { + repeated int32 ID = 1; + repeated float prob = 2; + } + ``` + +### Quick Start +You can run Friesian Serving Recommendation Framework using the official Docker images. + +You can follow the following steps to run the WnD demo. + +1. Pull docker image from dockerhub + ```bash + docker pull intelanalytics/friesian-grpc:0.0.2 + ``` + +2. Run & enter docker container + ```bash + docker run -itd --name friesian --net=host intelanalytics/friesian-grpc:0.0.2 + docker exec -it friesian bash + ``` + +3. Add vec_feature_user_prediction.parquet, vec_feature_item_prediction.parquet, wnd model, + wnd_item.parquet and wnd_user.parquet (You can check [the schema of the parquet files](#schema-of-the-parquet-files)) + +4. Start ranking service + ```bash + export OMP_NUM_THREADS=1 + java -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.ranking.RankingServer -c config_ranking.yaml > logs/inf.log 2>&1 & + ``` + +5. Start feature service for recommender service + ```bash + ./redis-5.0.5/src/redis-server & + java -Dspark.master=local[*] -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.feature.FeatureServer -c config_feature.yaml > logs/feature.log 2>&1 & + ``` + +6. Start feature service for recall service + ```bash + java -Dspark.master=local[*] -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.feature.FeatureServer -c config_feature_vec.yaml > logs/fea_recall.log 2>&1 & + ``` + +7. Start recall service + ```bash + java -Dspark.master=local[*] -Dspark.driver.maxResultSize=2G -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.recall.RecallServer -c config_recall.yaml > logs/vec.log 2>&1 & + ``` + +8. Start recommender service + ```bash + java -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.recommender.RecommenderServer -c config_recommender.yaml > logs/rec.log 2>&1 & + ``` + +9. Check if the services are running + ```bash + ps aux|grep friesian + ``` + You will see 5 processes start with 'java' + +10. Run client to test + ```bash + java -Dspark.master=local[*] -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.recommender.RecommenderMultiThreadClient -target localhost:8980 -dataDir wnd_user.parquet -k 50 -clientNum 4 -testNum 2 + ``` +11. Close services + ```bash + ps aux|grep friesian (find the service pid) + kill xxx (pid of the service which should be closed) + ``` + +### Schema of the parquet files + +#### The schema of the user and item embedding files +The embedding parquet files should contain at least 2 columns, id column and prediction column. +The id column should be IntegerType and the column name should be specified in the config files. +The prediction column should be DenseVector type, and you can transfer your existing embedding vectors using pyspark: +```python +from pyspark.sql import SparkSession +from pyspark.sql.functions import udf, col +from pyspark.ml.linalg import VectorUDT, DenseVector + +spark = SparkSession.builder \ + .master("local[*]") \ + .config("spark.driver.memory", "2g") \ + .getOrCreate() + +df = spark.read.parquet("data_path") + +def trans_densevector(data): + return DenseVector(data) + +vector_udf = udf(lambda x: trans_densevector(x), VectorUDT()) +# suppose the embedding column (ArrayType(FloatType,true)) is the existing user/item embedding. +df = df.withColumn("prediction", vector_udf(col("embedding"))) +df.write.parquet("output_file_path", mode="overwrite") +``` + +#### The schema of the recommendation model feature files +The feature parquet files should contain at least 2 columns, the id column and other feature columns. +The feature columns can be int, float, double, long and array of int, float, double and long. +Here is an example of the WideAndDeep model feature. +```bash ++-------------+--------+--------+----------+--------------------------------+---------------------------------+------------+-----------+---------+----------------------+-----------------------------+ +|present_media|language|tweet_id|tweet_type|engaged_with_user_follower_count|engaged_with_user_following_count|len_hashtags|len_domains|len_links|present_media_language|engaged_with_user_is_verified| ++-------------+--------+--------+----------+--------------------------------+---------------------------------+------------+-----------+---------+----------------------+-----------------------------+ +| 9| 43| 924| 2| 6| 3| 0.0| 0.1| 0.1| 45| 1| +| 0| 6| 4741724| 2| 3| 3| 0.0| 0.0| 0.0| 527| 0| ++-------------+--------+--------+----------+--------------------------------+---------------------------------+------------+-----------+---------+----------------------+-----------------------------+ +``` + +### The data schema in Redis +The user features, item features and user embedding vectors are saved in Redis. +The data saved in Redis is a key-value set. + +#### Key in Redis +The key in Redis consists of 3 parts: key prefix, data type, and data id. +- Key prefix is `redisKeyPrefix` specified in the feature service config file. +- Data type is one of `user` or `item`. +- Data id is the value of `userIDColumn` or `itemIDColumn`. +Here is an example of key: `2tower_user:29` + +#### Value in Redis +A row in the input parquet file will be converted to java array of object, then serialized into byte array, and encoded into Base64 string. + +#### Data schema entry +Every key prefix and data type combination has its data schema entry to save the corresponding column names. The key of the schema entry is `keyPrefix + dataType`, such as `2tower_user`. The value of the schema entry is a string of column names separated by `,`, such as `enaging_user_follower_count,enaging_user_following_count,enaging_user_is_verified`. + +### Config for different service +You can pass some important information to services using `-c config.yaml` +```bash +java -Dspark.master=local[*] -Dspark.driver.maxResultSize=2G -cp bigdl-friesian-serving-spark_2.4.6-0.14.0-SNAPSHOT.jar com.intel.analytics.bigdl.friesian.serving.recall.RecallServer -c config_recall.yaml +``` + +#### Ranking Service Config +Config with example: +```yaml +# Default: 8980, which port to create the server +servicePort: 8083 + +# Default: 0, open a port for prometheus monitoring tool, if set, user can check the +# performance using prometheus +monitorPort: 1234 + +# model path must be provided +modelPath: /home/yina/Documents/model/recys2021/wnd_813/recsys_wnd + +# default: null, savedmodel input list if the model is tf savedmodel. If not provided, the inputs +# of the savedmodel will be arranged in alphabetical order +savedModelInputs: serving_default_input_1:0, serving_default_input_2:0, serving_default_input_3:0, serving_default_input_4:0, serving_default_input_5:0, serving_default_input_6:0, serving_default_input_7:0, serving_default_input_8:0, serving_default_input_9:0, serving_default_input_10:0, serving_default_input_11:0, serving_default_input_12:0, serving_default_input_13:0 + +# default: 1, number of models used in inference service +modelParallelism: 4 +``` + +##### Feature Service Config +Config with example: +1. load data into redis. Search data from redis + ```yaml + ### Basic setting + # Default: 8980, which port to create the server + servicePort: 8082 + + # Default: null, open a port for prometheus monitoring tool, if set, user can check the + # performance using prometheus + monitorPort: 1235 + + # 'kv' or 'inference' default: kv + serviceType: kv + + # default: false, if need to load initial data to redis, set true + loadInitialData: true + + # default: "", prefix for redis key + redisKeyPrefix: + + # default: 0, item slot type on redis cluster. 0 means slot number use the default value 16384, 1 means all keys save to same slot, 2 means use the last character of id as hash tag. + redisClusterItemSlotType: 2 + + # default: null, if loadInitialData=true, initialUserDataPath or initialItemDataPath must be + # provided. Only support parquet file + initialUserDataPath: /home/yina/Documents/data/recsys/preprocess_output/wnd_user.parquet + initialItemDataPath: /home/yina/Documents/data/recsys/preprocess_output/wnd_exp1/wnd_item.parquet + + # default: null, if loadInitialData=true and initialUserDataPath != null, userIDColumn and + # userFeatureColumns must be provided + userIDColumn: enaging_user_id + userFeatureColumns: enaging_user_follower_count,enaging_user_following_count + + # default: null, if loadInitialData=true and initialItemDataPath != null, userIDColumn and + # userFeatureColumns must be provided + itemIDColumn: tweet_id + itemFeatureColumns: present_media, language, tweet_id, hashtags, present_links, present_domains, tweet_type, engaged_with_user_follower_count,engaged_with_user_following_count, len_hashtags, len_domains, len_links, present_media_language, tweet_id_engaged_with_user_id + + # default: null, user model path or item model path must be provided if serviceType + # contains 'inference'. If serviceType=kv, usermodelPath, itemModelPath and modelParallelism will + # be ignored + # userModelPath: + + # default: null, user model path or item model path must be provided if serviceType + # contains 'inference'. If serviceType=kv, usermodelPath, itemModelPath and modelParallelism will + # be ignored + # itemModelPath: + + # default: 1, number of models used for inference + # modelParallelism: + + ### Redis Configuration + # default: localhost:6379 + # redisUrl: + + # default: 256, JedisPoolMaxTotal + # redisPoolMaxTotal: + ``` + +2. load user features into redis. Get features from redis, use model at 'userModelPath' to do + inference and get the user embedding + ```yaml + ### Basic setting + # Default: 8980, which port to create the server + servicePort: 8085 + + # Default: null, open a port for prometheus monitoring tool, if set, user can check the + # performance using prometheus + monitorPort: 1236 + + # 'kv' or 'inference' default: kv + serviceType: kv, inference + + # default: false, if need to load initial data to redis, set true + loadInitialData: true + + # default: "" + redisKeyPrefix: 2tower_ + + # default: 0, item slot type on redis cluster. 0 means slot number use the default value 16384, 1 means all keys save to same slot, 2 means use the last character of id as hash tag. + redisClusterItemSlotType: 2 + + # default: null, if loadInitialData=true, initialDataPath must be provided. Only support parquet + # file + initialUserDataPath: /home/yina/Documents/data/recsys/preprocess_output/guoqiong/vec_feature_user.parquet + # initialItemDataPath: + + # default: null, if loadInitialData=true and initialUserDataPath != null, userIDColumn and + # userFeatureColumns must be provided + #userIDColumn: user + userIDColumn: enaging_user_id + userFeatureColumns: user + + # default: null, if loadInitialData=true and initialItemDataPath != null, userIDColumn and + # userFeatureColumns must be provided + # itemIDColumn: + # itemFeatureColumns: + + # default: null, user model path or item model path must be provided if serviceType + # includes 'inference'. If serviceType=kv, usermodelPath, itemModelPath and modelParallelism will + # be ignored + userModelPath: /home/yina/Documents/model/recys2021/2tower/guoqiong/user-model + + # default: null, user model path or item model path must be provided if serviceType + # contains 'inference'. If serviceType=kv, usermodelPath, itemModelPath and modelParallelism will + # be ignored + # itemModelPath: + + # default: 1, number of models used for inference + # modelParallelism: + + ### Redis Configuration + # default: localhost:6379 + # redisUrl: + + # default: 256, JedisPoolMaxTotal + # redisPoolMaxTotal: + ``` + +#### Recall Service Config +Config with example: + +1. load initial item vector from vec_feature_item.parquet and item-model to build faiss index. + ```yaml + # Default: 8980, which port to create the server + servicePort: 8084 + + # Default: null, open a port for prometheus monitoring tool, if set, user can check the + # performance using prometheus + monitorPort: 1238 + + # default: 128, the dimensionality of the embedding vectors + indexDim: 50 + + # default: false, if load saved index, set true + # loadSavedIndex: true + + # default: false, if true, the built index will be saved to indexPath. Ignored when + # loadSavedIndex=true + saveBuiltIndex: true + + # default: null, path to saved index path, must be provided if loadSavedIndex=true + indexPath: ./2tower_item_full.idx + + # default: false + getFeatureFromFeatureService: true + + # default: localhost:8980, feature service target + featureServiceURL: localhost:8085 + + itemIDColumn: tweet_id + itemFeatureColumns: item + + # default: null, user model path must be provided if getFeatureFromFeatureService=false + # userModelPath: + + # default: null, item model path must be provided if loadSavedIndex=false and initialDataPath is + # not orca predict result + itemModelPath: /home/yina/Documents/model/recys2021/2tower/guoqiong/item-model + + # default: null, Only support parquet file + initialDataPath: /home/yina/Documents/data/recsys/preprocess_output/guoqiong/vec_feature_item.parquet + + # default: 1, number of models used in inference service + modelParallelism: 1 + ``` + +2. load existing faiss index + ```yaml + # Default: 8980, which port to create the server + servicePort: 8084 + + # Default: null, open a port for prometheus monitoring tool, if set, user can check the + # performance using prometheus + monitorPort: 1238 + + # default: 128, the dimensionality of the embedding vectors + # indexDim: + + # default: false, if load saved index, set true + loadSavedIndex: true + + # default: null, path to saved index path, must be provided if loadSavedIndex=true + indexPath: ./2tower_item_full.idx + + # default: false + getFeatureFromFeatureService: true + + # default: localhost:8980, feature service target + featureServiceURL: localhost:8085 + + # itemIDColumn: + # itemFeatureColumns: + + # default: null, user model path must be provided if getFeatureFromFeatureService=false + # userModelPath: + + # default: null, item model path must be provided if loadSavedIndex=false and initialDataPath is + # not orca predict result + # itemModelPath: + + # default: null, Only support parquet file + # initialDataPath: + + # default: 1, number of models used in inference service + # modelParallelism: + ``` +#### Recommender Service Config +Config with example: + +```yaml + Default: 8980, which port to create the server + servicePort: 8980 + + # Default: null, open a port for prometheus monitoring tool, if set, user can check the + # performance using prometheus + monitorPort: 1237 + + # default: null, must be provided, item column name + itemIDColumn: tweet_id + +# default: null, must be provided, column names for inference, order related. +inferenceColumns: present_media_language, present_media, tweet_type, language, hashtags, present_links, present_domains, tweet_id_engaged_with_user_id, engaged_with_user_follower_count, engaged_with_user_following_count, enaging_user_follower_count, enaging_user_following_count, len_hashtags, len_domains, len_links + + # default: 0, if set, ranking service request will be divided +inferenceBatch: 0 + +# default: localhost:8980, recall service target +recallServiceURL: localhost:8084 + +# default: localhost:8980, feature service target +featureServiceURL: localhost:8082 + +# default: localhost:8980, inference service target +rankingServiceURL: localhost:8083 +``` + +### Run Java Client + +#### Generate proto java files +You should init a maven project and use proto files in [friesian gRPC project](https://github.com/analytics-zoo/friesian/tree/recsys-grpc/src/main/proto) +Make sure to add the following extensions and plugins in your pom.xml, and replace +*protocExecutable* with your own protoc executable. +```xml + + + + kr.motd.maven + os-maven-plugin + 1.6.2 + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 8 + 8 + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:1.37.0:exe:${os.detected.classifier} + /home/yina/Documents/protoc/bin/protoc + + + + + compile + compile-custom + + + + + + +``` +Then you can generate the gRPC files with +```bash +mvn clean install +``` +#### Call recommend service function using blocking stub +You can check the [Recommend service client example](https://github.com/analytics-zoo/friesian/blob/recsys-grpc/src/main/java/grpc/recommend/RecommendClient.java) on Github + +```java +import com.intel.analytics.bigdl.friesian.serving.grpc.generated.recommender.RecommenderGrpc; +import com.intel.analytics.bigdl.friesian.serving.grpc.generated.recommender.RecommenderProto.*; + +public class RecommendClient { + public static void main(String[] args) { + // Create a channel + ManagedChannel channel = ManagedChannelBuilder.forTarget(targetURL).usePlaintext().build(); + // Init a recommend service blocking stub + RecommenderGrpc.RecommenderBlockingStub blockingStub = RecommenderGrpc.newBlockingStub(channel); + // Construct a request + int[] userIds = new int[]{1}; + int candidateNum = 50; + int recommendNum = 10; + RecommendRequest.Builder request = RecommendRequest.newBuilder(); + for (int id : userIds) { + request.addID(id); + } + request.setCandidateNum(candidateNum); + request.setRecommendNum(recommendNum); + RecommendIDProbs recommendIDProbs = null; + try { + recommendIDProbs = blockingStub.getRecommendIDs(request.build()); + logger.info(recommendIDProbs.getIDProbListList()); + } catch (StatusRuntimeException e) { + logger.warn("RPC failed: " + e.getStatus().toString()); + } + } +} +``` + +### Run Python Client +Install the python packages listed below (you may encounter [pyspark error](https://stackoverflow.com/questions/58700384/how-to-fix-typeerror-an-integer-is-required-got-type-bytes-error-when-tryin) if you have python>=3.8 installed, try to downgrade to python<=3.7 and try again). +```bash +pip install jupyter notebook==6.1.4 grpcio grpcio-tools pandas fastparquet pyarrow +``` +After you activate your server successfully, you can + +#### Generate proto python files +Generate the files with +```bash +python -m grpc_tools.protoc -I../../protos --python_out= --grpc_python_out= /src/main/proto/*.proto +``` + +#### Call recommend service function using blocking stub +You can check the [Recommend service client example](https://github.com/analytics-zoo/friesian/blob/recsys-grpc/Serving/WideDeep/recommend_client.ipynb) on Github +```python +# create a channel +channel = grpc.insecure_channel('localhost:8980') +# create a recommend service stub +stub = recommender_pb2_grpc.RecommenderStub(channel) +request = recommender_pb2.RecommendRequest(recommendNum=10, candidateNum=50, ID=[36407]) +results = stub.getRecommendIDs(request) +print(results.IDProbList) + +``` +### Scale-out for Big Data +#### Redis Cluster +For large data set, Redis standalone has no enough memory to store whole data set, data sharding and Redis cluster are supported to handle it. You only need to set up a Redis Cluster to get it work. + +First, start N Redis instance on N machines. +``` +redis-server --cluster-enabled yes --cluster-config-file nodes-0.conf --cluster-node-timeout 50000 --appendonly no --save "" --logfile 0.log --daemonize yes --protected-mode no --port 6379 +``` +on each machine, choose a different port and start another M instances(M>=1), as the slave nodes of above N instances. + +Then, call initialization command on one machine, if you choose M=1 above, use `--cluster-replicas 1` +``` +redis-cli --cluster create 172.168.3.115:6379 172.168.3.115:6380 172.168.3.116:6379 172.168.3.116:6380 172.168.3.117:6379 172.168.3.117:6380 --cluster-replicas 1 +``` +and the Redis cluster would be ready. + +#### Scale Service with Envoy +Each of the services could be scaled out. It is recommended to use the same resource, e.g. single machine with same CPU and memory, to test which service is bottleneck. From empirical observations, vector search and inference usually be. + +##### How to run envoy: +1. [download](https://www.envoyproxy.io/docs/envoy/latest/start/install) and deploy envoy(below use docker as example): + * download: `docker pull envoyproxy/envoy-dev:21df5e8676a0f705709f0b3ed90fc2dbbd63cfc5` +2. run command: `docker run --rm -it -p 9082:9082 -p 9090:9090 envoyproxy/envoy-dev:79ade4aebd02cf15bd934d6d58e90aa03ef6909e --config-yaml "$(cat path/to/service-specific-envoy.yaml)" --parent-shutdown-time-s 1000000` +3. validate: run `netstat -tnlp` to see if the envoy process is listening to the corresponding port in the envoy config file. +4. For details on envoy and sample procedure, read [envoy](envoy.md). diff --git a/docs/readthedocs/source/doc/GetStarted/index.rst b/docs/readthedocs/source/doc/GetStarted/index.rst new file mode 100644 index 00000000..44ff4eb3 --- /dev/null +++ b/docs/readthedocs/source/doc/GetStarted/index.rst @@ -0,0 +1,6 @@ +User Guide +========================= + + +Getting Started +=========================================== \ No newline at end of file diff --git a/docs/readthedocs/source/doc/GetStarted/install.rst b/docs/readthedocs/source/doc/GetStarted/install.rst new file mode 100644 index 00000000..2ecc9bab --- /dev/null +++ b/docs/readthedocs/source/doc/GetStarted/install.rst @@ -0,0 +1,2 @@ +Install Locally +========================= \ No newline at end of file diff --git a/docs/readthedocs/source/doc/GetStarted/paper.md b/docs/readthedocs/source/doc/GetStarted/paper.md new file mode 100644 index 00000000..cfd2fb4d --- /dev/null +++ b/docs/readthedocs/source/doc/GetStarted/paper.md @@ -0,0 +1,28 @@ +# Paper + + +## Paper + +* Dai, Jason Jinquan, et al. "BigDL 2.0: Seamless Scaling of AI Pipelines from Laptops to Distributed Cluster." Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022. [paper](https://arxiv.org/ftp/arxiv/papers/2204/2204.01715.pdf) [video]() [demo]() + +* Dai, Jason Jinquan, et al. "BigDL: A distributed deep learning framework for big data." Proceedings of the ACM Symposium on Cloud Computing. 2019. [paper](https://arxiv.org/abs/1804.05839) + + + + +## Citing +If you've found BigDL useful for your project, you may cite the [paper](https://arxiv.org/abs/1804.05839) as follows: + +``` +@inproceedings{SOCC2019_BIGDL, + title={BigDL: A Distributed Deep Learning Framework for Big Data}, + author={Dai, Jason (Jinquan) and Wang, Yiheng and Qiu, Xin and Ding, Ding and Zhang, Yao and Wang, Yanzhang and Jia, Xianyan and Zhang, Li (Cherry) and Wan, Yan and Li, Zhichao and Wang, Jiao and Huang, Shengsheng and Wu, Zhongyuan and Wang, Yang and Yang, Yuhao and She, Bowen and Shi, Dongjie and Lu, Qi and Huang, Kai and Song, Guoqiong}, + booktitle={Proceedings of the ACM Symposium on Cloud Computing}, + publisher={Association for Computing Machinery}, + pages={50--60}, + year={2019}, + series={SoCC'19}, + doi={10.1145/3357223.3362707}, + url={https://arxiv.org/pdf/1804.05839.pdf} +} +``` \ No newline at end of file diff --git a/docs/readthedocs/source/doc/GetStarted/usecase.rst b/docs/readthedocs/source/doc/GetStarted/usecase.rst new file mode 100644 index 00000000..b405af0e --- /dev/null +++ b/docs/readthedocs/source/doc/GetStarted/usecase.rst @@ -0,0 +1,2 @@ +Use Cases +============================ \ No newline at end of file diff --git a/docs/readthedocs/source/doc/GetStarted/videos.md b/docs/readthedocs/source/doc/GetStarted/videos.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/readthedocs/source/doc/Nano/QuickStart/hpo.rst b/docs/readthedocs/source/doc/Nano/Overview/hpo.rst similarity index 99% rename from docs/readthedocs/source/doc/Nano/QuickStart/hpo.rst rename to docs/readthedocs/source/doc/Nano/Overview/hpo.rst index ced5a0fa..c48135d9 100644 --- a/docs/readthedocs/source/doc/Nano/QuickStart/hpo.rst +++ b/docs/readthedocs/source/doc/Nano/Overview/hpo.rst @@ -1,4 +1,4 @@ -AutoML Overview +AutoML *************** Nano provides built-in AutoML support through hyperparameter optimization. diff --git a/docs/readthedocs/source/doc/Nano/Overview/index.rst b/docs/readthedocs/source/doc/Nano/Overview/index.rst new file mode 100644 index 00000000..14cdd4c0 --- /dev/null +++ b/docs/readthedocs/source/doc/Nano/Overview/index.rst @@ -0,0 +1,8 @@ +Nano Key Features +================================ + +* `PyTorch Training `_ +* `PyTorch Inference `_ +* `Tensorflow Training `_ +* `Tensorflow Inference `_ +* `AutoML `_ \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Nano/Overview/install.md b/docs/readthedocs/source/doc/Nano/Overview/install.md new file mode 100644 index 00000000..e4fc151c --- /dev/null +++ b/docs/readthedocs/source/doc/Nano/Overview/install.md @@ -0,0 +1,36 @@ +# Nano Installation + +Note: For windows users, we recommend using Windows Subsystem for Linux 2 (WSL2) to run BigDL-Nano. Please refer to [Nano Windows install guide](../Howto/windows_guide.md) for instructions. + + +BigDL-Nano can be installed using pip and we recommend installing BigDL-Nano in a conda environment. + +For PyTorch Users, you can install bigdl-nano along with some dependencies specific to PyTorch using the following commands. + +```bash +conda create -n env +conda activate env +pip install bigdl-nano[pytorch] +``` + +For TensorFlow users, you can install bigdl-nano along with some dependencies specific to TensorFlow using the following commands. + +```bash +conda create -n env +conda activate env +pip install bigdl-nano[tensorflow] +``` + +After installing bigdl-nano, you can run the following command to setup a few environment variables. + +```bash +source bigdl-nano-init +``` + +The `bigdl-nano-init` scripts will export a few environment variable according to your hardware to maximize performance. + +In a conda environment, `source bigdl-nano-init` will also be added to `$CONDA_PREFIX/etc/conda/activate.d/`, which will automaticly run when you activate your current environment. + +In a pure pip environment, you need to run `source bigdl-nano-init` every time you open a new shell to get optimal performance and run `source bigdl-nano-unset-env` if you want to unset these environment variables. + +--- \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Nano/Overview/nano.md b/docs/readthedocs/source/doc/Nano/Overview/nano.md index 480a9c31..f6280bde 100644 --- a/docs/readthedocs/source/doc/Nano/Overview/nano.md +++ b/docs/readthedocs/source/doc/Nano/Overview/nano.md @@ -1,49 +1,11 @@ -# Nano User Guide +# Nano in 5 minutes -## **1. Overview** +BigDL-Nano is a Python package to transparently accelerate PyTorch and TensorFlow applications on Intel hardware. It provides a unified and easy-to-use API for several optimization techniques and tools, so that users can only apply a few lines of code changes to make their PyTorch or TensorFlow code run faster. -BigDL Nano is a Python package to transparently accelerate PyTorch and TensorFlow applications on Intel hardware. It provides a unified and easy-to-use API for several optimization techniques and tools, so that users can only apply a few lines of code changes to make their PyTorch or TensorFlow code run faster. +---- ---- -## **2. Install** -Note: For windows users, we recommend using Windows Subsystem for Linux 2 (WSL2) to run BigDL-Nano. Please refer to [Nano Windows install guide](../Howto/windows_guide.md) for instructions. - -BigDL-Nano can be installed using pip and we recommend installing BigDL-Nano in a conda environment. - -For PyTorch Users, you can install bigdl-nano along with some dependencies specific to PyTorch using the following commands. - -```bash -conda create -n env -conda activate env -pip install bigdl-nano[pytorch] -``` - -For TensorFlow users, you can install bigdl-nano along with some dependencies specific to TensorFlow using the following commands. - -```bash -conda create -n env -conda activate env -pip install bigdl-nano[tensorflow] -``` - -After installing bigdl-nano, you can run the following command to setup a few environment variables. - -```bash -source bigdl-nano-init -``` - -The `bigdl-nano-init` scripts will export a few environment variable according to your hardware to maximize performance. - -In a conda environment, `source bigdl-nano-init` will also be added to `$CONDA_PREFIX/etc/conda/activate.d/`, which will automaticly run when you activate your current environment. - -In a pure pip environment, you need to run `source bigdl-nano-init` every time you open a new shell to get optimal performance and run `source bigdl-nano-unset-env` if you want to unset these environment variables. - ---- - -## **3. Get Started** - -#### **3.1 PyTorch** +### **PyTorch Bite-sized Example** BigDL-Nano supports both PyTorch and PyTorch Lightning models and most optimizations require only changing a few "import" lines in your code and adding a few flags. @@ -74,7 +36,8 @@ MyNano(use_ipex=True, num_processes=2).train() For more details on the BigDL-Nano's PyTorch usage, please refer to the [PyTorch Training](../QuickStart/pytorch_train.md) and [PyTorch Inference](../QuickStart/pytorch_inference.md) page. -### **3.2 TensorFlow** + +### **TensorFlow Bite-sized Example** BigDL-Nano supports `tensorflow.keras` API and most optimizations require only changing a few "import" lines in your code and adding a few flags. @@ -104,4 +67,4 @@ model.compile(optimizer='adam', model.fit(x_train, y_train, epochs=5, num_processes=4) ``` -For more details on the BigDL-Nano's PyTorch usage, please refer to the [TensorFlow Training](../QuickStart/tensorflow_train.md) and [TensorFlow Inference](../QuickStart/tensorflow_inference.md) page. +For more details on the BigDL-Nano's Tensorflow usage, please refer to the [TensorFlow Training](../QuickStart/tensorflow_train.md) and [TensorFlow Inference](../QuickStart/tensorflow_inference.md) page. diff --git a/docs/readthedocs/source/doc/Nano/QuickStart/pytorch_inference.md b/docs/readthedocs/source/doc/Nano/Overview/pytorch_inference.md similarity index 98% rename from docs/readthedocs/source/doc/Nano/QuickStart/pytorch_inference.md rename to docs/readthedocs/source/doc/Nano/Overview/pytorch_inference.md index ae3d591c..04371ee2 100644 --- a/docs/readthedocs/source/doc/Nano/QuickStart/pytorch_inference.md +++ b/docs/readthedocs/source/doc/Nano/Overview/pytorch_inference.md @@ -1,4 +1,4 @@ -# BigDL-Nano PyTorch Inference Overview +# PyTorch Inference BigDL-Nano provides several APIs which can help users easily apply optimizations on inference pipelines to improve latency and throughput. Currently, performance accelerations are achieved by integrating extra runtimes as inference backend engines or using quantization methods on full-precision trained models to reduce computation during inference. InferenceOptimizer (`bigdl.nano.pytorch.InferenceOptimizer`) provides the APIs for all optimizations that you need for inference. @@ -70,7 +70,7 @@ y_hat = ort_model(x) trainer.validate(ort_model, dataloader) trainer.test(ort_model, dataloader) trainer.predict(ort_model, dataloader) -# note that `ort_model` is not trainable any more, so you can't use like +# note that `ort_model` is not trainable any more, so you can't use like # trainer.fit(ort_model, dataloader) # this is illegal ``` ### OpenVINO Acceleration @@ -93,7 +93,7 @@ trainer = Trainer() trainer.validate(ort_model, dataloader) trainer.test(ort_model, dataloader) trainer.predict(ort_model, dataloader) -# note that `ort_model` is not trainable any more, so you can't use like +# note that `ort_model` is not trainable any more, so you can't use like # trainer.fit(ort_model, dataloader) # this is illegal ``` @@ -122,7 +122,7 @@ trainer.validate(q_model, dataloader) trainer.test(q_model, dataloader) trainer.predict(q_model, dataloader) ``` -This is a most basic usage to quantize a model with defaults, INT8 precision, and without search tuning space to control accuracy drop. +This is a most basic usage to quantize a model with defaults, INT8 precision, and without search tuning space to control accuracy drop. **Quantization with ONNXRuntime accelerator** @@ -146,7 +146,7 @@ Using `accelerator='onnxruntime'` actually equals to converting the model from P ort_model = InferenceOptimizer.trace(model, accelerator='onnruntime', input_sample=x): ort_q_model = InferenceOptimizer.quantize(ort_model, accelerator='onnxruntime', calib_dataloader=dataloader) -# run inference with transparent acceleration +# run inference with transparent acceleration y_hat = ort_q_model(x) trainer.validate(ort_q_model, dataloader) trainer.test(ort_q_model, dataloader) @@ -174,7 +174,7 @@ Same as using ONNXRuntime accelerator, it equals to converting the model from Py ov_model = InferenceOptimizer.trace(model, accelerator='openvino', input_sample=x): ov_q_model = InferenceOptimizer.quantize(ov_model, accelerator='onnxruntime', calib_dataloader=dataloader) -# run inference with transparent acceleration +# run inference with transparent acceleration y_hat = ov_q_model(x) trainer.validate(ov_q_model, dataloader) trainer.test(ov_q_model, dataloader) diff --git a/docs/readthedocs/source/doc/Nano/QuickStart/pytorch_train.md b/docs/readthedocs/source/doc/Nano/Overview/pytorch_train.md similarity index 99% rename from docs/readthedocs/source/doc/Nano/QuickStart/pytorch_train.md rename to docs/readthedocs/source/doc/Nano/Overview/pytorch_train.md index 55a75dd9..2ccb2e0e 100644 --- a/docs/readthedocs/source/doc/Nano/QuickStart/pytorch_train.md +++ b/docs/readthedocs/source/doc/Nano/Overview/pytorch_train.md @@ -1,4 +1,4 @@ -# BigDL-Nano PyTorch Training Overview +# PyTorch Training BigDL-Nano can be used to accelerate PyTorch or PyTorch-Lightning applications on training workloads. The optimizations in BigDL-Nano are delivered through an extended version of PyTorch-Lightning `Trainer`. These optimizations are either enabled by default or can be easily turned on by setting a parameter or calling a method. diff --git a/docs/readthedocs/source/doc/Nano/QuickStart/tensorflow_inference.md b/docs/readthedocs/source/doc/Nano/Overview/tensorflow_inference.md similarity index 97% rename from docs/readthedocs/source/doc/Nano/QuickStart/tensorflow_inference.md rename to docs/readthedocs/source/doc/Nano/Overview/tensorflow_inference.md index e2180eb6..610649c5 100644 --- a/docs/readthedocs/source/doc/Nano/QuickStart/tensorflow_inference.md +++ b/docs/readthedocs/source/doc/Nano/Overview/tensorflow_inference.md @@ -1,12 +1,13 @@ -# BigDL-Nano TensorFlow Inference Overview -BigDL-Nano provides several APIs which can help users easily apply optimizations on inference pipelines to improve latency and throughput. Currently, performance accelerations are achieved by integrating extra runtimes as inference backend engines or using quantization methods on full-precision trained models to reduce computation during inference. Keras Model (`bigdl.nano.tf.keras.Model`) and Sequential (`bigdl.nano.tf.keras.Sequential`) provides the APIs for all optimizations that you need for inference. +# TensorFlow Inference + +BigDL-Nano provides several APIs which can help users easily apply optimizations on inference pipelines to improve latency and throughput. Currently, performance accelerations are achieved by integrating extra runtimes as inference backend engines or using quantization methods on full-precision trained models to reduce computation during inference. Keras Model (`bigdl.nano.tf.keras.Model`) and Sequential (`bigdl.nano.tf.keras.Sequential`) provides the APIs for all optimizations that you need for inference. For quantization, BigDL-Nano provides only post-training quantization in `Model.quantize()` for users to infer with models of 8-bit precision. Quantization-Aware Training is not available for now. Model conversion to 16-bit like BF16, and FP16 will be coming soon. Before you go ahead with these APIs, you have to make sure BigDL-Nano is correctly installed for TensorFlow. If not, please follow [this](../Overview/nano.md) to set up your environment. ## Quantization -Quantization is widely used to compress models to a lower precision, which not only reduces the model size but also accelerates inference. BigDL-Nano provides `Model.quantize()` API for users to quickly obtain a quantized model with accuracy control by specifying a few arguments. `Sequential` has similar usage, so we will only show how to use an instance of `Model` to enable quantization pipeline here. +Quantization is widely used to compress models to a lower precision, which not only reduces the model size but also accelerates inference. BigDL-Nano provides `Model.quantize()` API for users to quickly obtain a quantized model with accuracy control by specifying a few arguments. `Sequential` has similar usage, so we will only show how to use an instance of `Model` to enable quantization pipeline here. To use INC as your quantization engine, you can choose accelerator as `None` or `'onnxruntime'`. Otherwise, `accelerator='openvino'` means using OpenVINO POT to do quantization. diff --git a/docs/readthedocs/source/doc/Nano/QuickStart/tensorflow_train.md b/docs/readthedocs/source/doc/Nano/Overview/tensorflow_train.md similarity index 97% rename from docs/readthedocs/source/doc/Nano/QuickStart/tensorflow_train.md rename to docs/readthedocs/source/doc/Nano/Overview/tensorflow_train.md index b4a9d304..31cfb83e 100644 --- a/docs/readthedocs/source/doc/Nano/QuickStart/tensorflow_train.md +++ b/docs/readthedocs/source/doc/Nano/Overview/tensorflow_train.md @@ -1,4 +1,4 @@ -# BigDL-Nano TensorFlow Training Overview +# TensorFlow Training BigDL-Nano can be used to accelerate TensorFlow Keras applications on training workloads. The optimizations in BigDL-Nano are delivered through BigDL-Nano's `Model` and `Sequential` classes, which have identical APIs with `tf.keras.Model` and `tf.keras.Sequential`. For most cases, you can just replace your `tf.keras.Model` with `bigdl.nano.tf.keras.Model` and `tf.keras.Sequential` with `bigdl.nano.tf.keras.Sequential` to benefit from BigDL-Nano. @@ -38,6 +38,6 @@ model.compile(optimizer='adam', model.fit(train_ds, epochs=3, validation_data=val_ds, num_processes=2) ``` -Note that, different from the conventions in [BigDL-Nano PyTorch multi-instance training](./pytorch_train.html#multi-instance-training), the effective batch size will not change in TensorFlow multi-instance training, which means it is still the batch size you specify in your dataset. This is because TensorFlow's `MultiWorkerMirroredStrategy` will try to split the batch into multiple sub-batches for different workers. We chose this behavior to match the semantics of TensorFlow distributed training. +Note that, different from the conventions in [BigDL-Nano PyTorch multi-instance training](./pytorch_train.html#multi-instance-training), the effective batch size will not change in TensorFlow multi-instance training, which means it is still the batch size you specify in your dataset. This is because TensorFlow's `MultiWorkerMirroredStrategy` will try to split the batch into multiple sub-batches for different workers. We chose this behavior to match the semantics of TensorFlow distributed training. When you do want to increase your effective `batch_size`, you can do so by directly changing it in your dataset definition and you may also want to gradually increase the learning rate linearly to the `batch_size`, as described in this [paper](https://arxiv.org/abs/1706.02677) published by Facebook. \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Nano/Overview/userguide.rst b/docs/readthedocs/source/doc/Nano/Overview/userguide.rst new file mode 100644 index 00000000..e69de29b diff --git a/docs/readthedocs/source/doc/Nano/index.rst b/docs/readthedocs/source/doc/Nano/index.rst new file mode 100644 index 00000000..e5b6b57e --- /dev/null +++ b/docs/readthedocs/source/doc/Nano/index.rst @@ -0,0 +1,63 @@ +BigDL-Nano +========================= + +**BigDL-Nano** (or **Nano** for short) is a Python package to transparently accelerate PyTorch and TensorFlow applications on Intel hardware. It provides a unified and easy-to-use API for several optimization techniques and tools, so that users can only apply a few lines of code changes to make their PyTorch or TensorFlow code run faster. + +------- + + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + Documents in these sections helps you getting started quickly with Nano. + + +++ + :bdg-link:`Nano in 5 minutes <./Overview/nano.html>` | + :bdg-link:`Installation <./Overview/install.html>` | + :bdg-link:`Tutorials <./QuickStart/index.html>` + + .. grid-item-card:: + + **Key Features Guide** + ^^^ + + Each guide in this section provides you with in-depth information, concepts and knowledges about Nano key features. + + +++ + + :bdg:`PyTorch` :bdg-link:`Infer <./Overview/pytorch_inference.html>` :bdg-link:`Train <./Overview/pytorch_train.html>` | + :bdg:`TensorFlow` :bdg-link:`Infer <./Overview/tensorflow_inference.html>` :bdg-link:`Train <./Overview/tensorflow_train.html>` + + .. grid-item-card:: + + **How-to Guide** + ^^^ + + How-to Guide provides bite-sized, actionable examples of how to use specific Nano features, different from our tutorials + which are full-length examples each implementing a full usage scenario. + + +++ + + :bdg-link:`How-to-Guide <./Howto/index.html>` + + .. grid-item-card:: + + **API Document** + ^^^ + + API Document provides detailed description of Nano APIs. + + +++ + + :bdg-link:`API Document <../PythonAPI/Nano/index.html>` + + +.. toctree:: + :hidden: + + BigDL-Nano Document diff --git a/docs/readthedocs/source/doc/Orca/Overview/distributed-tuning.md b/docs/readthedocs/source/doc/Orca/Overview/distributed-tuning.md index 56379a2d..3a4bfc0d 100644 --- a/docs/readthedocs/source/doc/Orca/Overview/distributed-tuning.md +++ b/docs/readthedocs/source/doc/Orca/Overview/distributed-tuning.md @@ -2,25 +2,8 @@ --- -**Orca `AutoEstimator` provides similar APIs as Orca `Estimator` for distributed hyper-parameter tuning.** +**Orca `AutoEstimator` provides similar APIs as Orca `Estimator` for distributed hyper-parameter tuning.** -### **Install** -We recommend using [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to prepare the Python environment. -```bash -conda create -n bigdl-orca-automl python=3.7 # "bigdl-orca-automl" is conda environment name, you can use any name you like. -conda activate bigdl-orca-automl -pip install bigdl-orca[automl] -```` -You can install the latest release version of BigDL Orca as follows: -```bash -pip install --pre --upgrade bigdl-orca[automl] -``` -_Note that with extra key of [automl], `pip` will automatically install the additional dependencies for distributed hyper-parameter tuning, -including `ray[tune]==1.9.2`, `scikit-learn`, `tensorboard`, `xgboost`._ - -To use [Pytorch Estimator](#pytorch-autoestimator), you need to install Pytorch with `pip install torch==1.8.1`. - -To use [TensorFlow/Keras AutoEstimator](#tensorflow-keras-autoestimator), you need to install Tensorflow with `pip install tensorflow==1.15.0`. ### **1. AutoEstimator** @@ -28,11 +11,11 @@ To use [TensorFlow/Keras AutoEstimator](#tensorflow-keras-autoestimator), you ne To perform distributed hyper-parameter tuning, user can first create an Orca `AutoEstimator` from standard TensorFlow Keras or PyTorch model, and then call `AutoEstimator.fit`. Under the hood, the Orca `AutoEstimator` generates different trials and schedules them on each mode in the cluster. Each trial runs a different combination of hyper parameters, sampled from the user-desired hyper-parameter space. -HDFS is used to save temporary results of each trial and all the results will be finally transferred to driver for further analysis. +HDFS is used to save temporary results of each trial and all the results will be finally transferred to driver for further analysis. ### **2. Pytorch AutoEstimator** -User could pass *Creator Function*s, including *Data Creator Function*, *Model Creator Function* and *Optimizer Creator Function* to `AutoEstimator` for training. +User could pass *Creator Function*s, including *Data Creator Function*, *Model Creator Function* and *Optimizer Creator Function* to `AutoEstimator` for training. The *Creator Function*s should take a parameter of `config` as input and get the hyper-parameter values from `config` to enable hyper parameter search. @@ -64,7 +47,7 @@ class LeNet(nn.Module): self.conv2 = nn.Conv2d(20, 50, 5, 1) self.fc1 = nn.Linear(4*4*50, fc1_hidden_size) self.fc2 = nn.Linear(fc1_hidden_size, 10) - + def forward(self, x): pass @@ -75,7 +58,7 @@ def model_creator(config): ``` #### **2.3 Optimizer Creator Function** -*Optimizer Creator Function* takes `model` and `config` as input, and returns a `torch.optim.Optimizer` object. +*Optimizer Creator Function* takes `model` and `config` as input, and returns a `torch.optim.Optimizer` object. ```python import torch def optim_creator(model, config): @@ -170,7 +153,7 @@ search_space = { ``` #### **4.2 Advanced Search Algorithms** -Beside grid search and random search, user could also choose to use some advanced hyper-parameter optimization methods, +Beside grid search and random search, user could also choose to use some advanced hyper-parameter optimization methods, such as [Ax](https://ax.dev/), [Bayesian Optimization](https://github.com/fmfn/BayesianOptimization), [Scikit-Optimize](https://scikit-optimize.github.io), etc. We supported all *Search Algorithms* in [Ray Tune](https://docs.ray.io/en/master/index.html). View the [Ray Tune Search Algorithms](https://docs.ray.io/en/master/tune/api_docs/suggestion.html) for more details. Note that you should install the dependency for your search algorithm manually. @@ -207,7 +190,7 @@ We support all *Schedulers* in [Ray Tune](https://docs.ray.io/en/master/index.ht User can pass the *Scheduler* name to `scheduler` in `AutoEstimator.fit`. The *Scheduler* names supported are "fifo", "hyperband", "async_hyperband", "median_stopping_rule", "hb_bohb", "pbt", "pbt_replay". The default `scheduler` is "fifo", which just runs trials in submission order. -See examples below about how to use *Scheduler* in `AutoEstimator`. +See examples below about how to use *Scheduler* in `AutoEstimator`. ```python scheduler_params = dict( max_t=50, diff --git a/docs/readthedocs/source/doc/Orca/Overview/getstarted.rst b/docs/readthedocs/source/doc/Orca/Overview/getstarted.rst new file mode 100644 index 00000000..cf74e658 --- /dev/null +++ b/docs/readthedocs/source/doc/Orca/Overview/getstarted.rst @@ -0,0 +1,2 @@ +Orca Key Features +================================= \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Orca/Overview/index.rst b/docs/readthedocs/source/doc/Orca/Overview/index.rst new file mode 100644 index 00000000..c2eec58f --- /dev/null +++ b/docs/readthedocs/source/doc/Orca/Overview/index.rst @@ -0,0 +1,8 @@ +Orca Key Features +================================= + +* `Orca Context `_ +* `Distributed Data Processing `_ +* `Distributed Training and Inference `_ +* `Distributed Hyper Parameter Tuning `_ +* `RayOnSpark `_ \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Orca/Overview/install.md b/docs/readthedocs/source/doc/Orca/Overview/install.md new file mode 100644 index 00000000..9f6a7bea --- /dev/null +++ b/docs/readthedocs/source/doc/Orca/Overview/install.md @@ -0,0 +1,45 @@ +# Installation + + +## To use Distributed Data processing, training, and/or inference +We recommend using [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to prepare the Python environment. +```bash +conda create -n py37 python=3.7 # "py37" is conda environment name, you can use any name you like. +conda activate py37 +pip install bigdl-orca +``` + +You can install bigdl-orca nightly build version using +```bash +pip install --pre --upgrade bigdl-orca +``` + +## To use RayOnSpark + +There're some additional dependencies required for running [RayOnSpark](ray.md). Use extra key `[ray]` to install. + +```bash +pip install bigdl-orca[ray] +``` + +or to install nightly build, use +```bash +pip install --pre --upgrade bigdl-orca[ray] +``` + +## To use Orca AutoML + +There're some additional dependencies required for Orca AutoML support. Use extra key `[automl]` to install. + +```bash +pip install bigdl-orca[automl] +```` + + +_Note that with extra key of [automl], `pip` will automatically install the additional dependencies for distributed hyper-parameter tuning, +including `ray[tune]==1.9.2`, `scikit-learn`, `tensorboard`, `xgboost`._ + +To use [Pytorch Estimator](#pytorch-autoestimator), you need to install Pytorch with `pip install torch==1.8.1`. + +To use [TensorFlow/Keras AutoEstimator](#tensorflow-keras-autoestimator), you need to install Tensorflow with `pip install tensorflow==1.15.0`. + diff --git a/docs/readthedocs/source/doc/Orca/Overview/known_issues.md b/docs/readthedocs/source/doc/Orca/Overview/known_issues.md index fdb70f34..28260f09 100644 --- a/docs/readthedocs/source/doc/Orca/Overview/known_issues.md +++ b/docs/readthedocs/source/doc/Orca/Overview/known_issues.md @@ -6,7 +6,7 @@ This error occurs while running Orca TF2 Estimator with spark backend, which may because the previous pyspark tensorflow job was not cleaned completely. You can retry later or you can set spark config `spark.python.worker.reuse=false` in your application. -If you are using `init_orca_context(cluster_mode="yarn-client")`: +If you are using `init_orca_context(cluster_mode="yarn-client")`: ``` conf = {"spark.python.worker.reuse": "false"} init_orca_context(cluster_mode="yarn-client", conf=conf) @@ -19,10 +19,10 @@ If you are using `init_orca_context(cluster_mode="yarn-client")`: ### **RuntimeError: Inter op parallelism cannot be modified after initialization** This error occurs if you build your TensorFlow model on the driver rather than on workers. You should build the complete model in `model_creator` which runs on each worker node. You can refer to the following examples: - + **Wrong Example** ``` - model = ... + model = ... def model_creator(config): model.compile(...) @@ -85,3 +85,43 @@ To solve this issue, you need to set the path of `libhdfs.so` in Cloudera to the # For yarn-cluster mode spark-submit --conf spark.executorEnv.ARROW_LIBHDFS_DIR=/opt/cloudera/parcels/CDH-5.15.2-1.cdh5.15.2.p0.3/lib64 \ --conf spark.yarn.appMasterEnv.ARROW_LIBHDFS_DIR=/opt/cloudera/parcels/CDH-5.15.2-1.cdh5.15.2.p0.3/lib64 + + +### **Spark Dynamic Allocation** + +By design, BigDL does not support Spark Dynamic Allocation mode, and needs to allocate fixed resources for deep learning model training. Thus if your environment has already configured Spark Dynamic Allocation, or stipulated that Spark Dynamic Allocation must be used, you may encounter the following error: + +> **requirement failed: Engine.init: spark.dynamicAllocation.maxExecutors and spark.dynamicAllocation.minExecutors must be identical in dynamic allocation for BigDL** +> + +Here we provide a workaround for running BigDL under Spark Dynamic Allocation mode. + +For `spark-submit` cluster mode, the first solution is to disable the Spark Dynamic Allocation mode in `SparkConf` when you submit your application as follows: + +```bash +spark-submit --conf spark.dynamicAllocation.enabled=false +``` + +Otherwise, if you can not set this configuration due to your cluster settings, you can set `spark.dynamicAllocation.minExecutors` to be equal to `spark.dynamicAllocation.maxExecutors` as follows: + +```bash +spark-submit --conf spark.dynamicAllocation.enabled=true \ + --conf spark.dynamicAllocation.minExecutors 2 \ + --conf spark.dynamicAllocation.maxExecutors 2 +``` + +For other cluster modes, such as `yarn` and `k8s`, our program will initiate `SparkContext` for you, and the Spark Dynamic Allocation mode is disabled by default. Thus, generally you wouldn't encounter such problem. + +If you are using Spark Dynamic Allocation, you have to disable barrier execution mode at the very beginning of your application as follows: + +```python +from bigdl.orca import OrcaContext + +OrcaContext.barrier_mode = False +``` + +For Spark Dynamic Allocation mode, you are also recommended to manually set `num_ray_nodes` and `ray_node_cpu_cores` equal to `spark.dynamicAllocation.minExecutors` and `spark.executor.cores` respectively. You can specify `num_ray_nodes` and `ray_node_cpu_cores` in `init_orca_context` as follows: + +```python +init_orca_context(..., num_ray_nodes=2, ray_node_cpu_cores=4) +``` diff --git a/docs/readthedocs/source/doc/Orca/Overview/orca.md b/docs/readthedocs/source/doc/Orca/Overview/orca.md index 69857d24..0f03929b 100644 --- a/docs/readthedocs/source/doc/Orca/Overview/orca.md +++ b/docs/readthedocs/source/doc/Orca/Overview/orca.md @@ -1,30 +1,12 @@ -# The Orca Library +# Orca in 5 minutes -## 1. Overview +### Overview Most AI projects start with a Python notebook running on a single laptop; however, one usually needs to go through a mountain of pains to scale it to handle larger data set in a distributed fashion. The _**Orca**_ library seamlessly scales out your single node Python notebook across large clusters (so as to process distributed Big Data). -## 2. Install -We recommend using [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to prepare the Python environment. -```bash -conda create -n py37 python=3.7 # "py37" is conda environment name, you can use any name you like. -conda activate py37 -pip install bigdl-orca -``` +--- -When installing bigdl-orca with pip, you can specify the extras key `[ray]` to additionally install the additional dependencies -essential for running [RayOnSpark](../../Ray/Overview/ray.md) -```bash -pip install bigdl-orca[ray] -``` - -You can install bigdl-orca nightly release version using -```bash -pip install --pre --upgrade bigdl-orca -pip install --pre --upgrade bigdl-orca[ray] -``` - -## 3. Run +### **Tensorflow Bite-sized Example** This section uses TensorFlow 1.15, and you should install TensorFlow before running this example: ```bash @@ -37,7 +19,7 @@ First, initialize [Orca Context](orca-context.md): from bigdl.orca import init_orca_context, OrcaContext # cluster_mode can be "local", "k8s" or "yarn" -sc = init_orca_context(cluster_mode="local", cores=4, memory="10g", num_nodes=1) +sc = init_orca_context(cluster_mode="local", cores=4, memory="10g", num_nodes=1) ``` Next, perform [data-parallel processing in Orca](data-parallel-processing.md) (supporting standard Spark Dataframes, TensorFlow Dataset, PyTorch DataLoader, Pandas, etc.): @@ -47,7 +29,7 @@ from pyspark.sql.functions import array spark = OrcaContext.get_spark_session() df = spark.read.parquet(file_path) -df = df.withColumn('user', array('user')) \ +df = df.withColumn('user', array('user')) \ .withColumn('item', array('item')) ``` @@ -57,24 +39,19 @@ Finally, use [sklearn-style Estimator APIs in Orca](distributed-training-inferen from tensorflow import keras from bigdl.orca.learn.tf.estimator import Estimator -user = keras.layers.Input(shape=[1]) -item = keras.layers.Input(shape=[1]) -feat = keras.layers.concatenate([user, item], axis=1) -predictions = keras.layers.Dense(2, activation='softmax')(feat) -model = keras.models.Model(inputs=[user, item], outputs=predictions) -model.compile(optimizer='rmsprop', - loss='sparse_categorical_crossentropy', +user = keras.layers.Input(shape=[1]) +item = keras.layers.Input(shape=[1]) +feat = keras.layers.concatenate([user, item], axis=1) +predictions = keras.layers.Dense(2, activation='softmax')(feat) +model = keras.models.Model(inputs=[user, item], outputs=predictions) +model.compile(optimizer='rmsprop', + loss='sparse_categorical_crossentropy', metrics=['accuracy']) -est = Estimator.from_keras(keras_model=model) -est.fit(data=df, - batch_size=64, - epochs=4, - feature_cols=['user', 'item'], +est = Estimator.from_keras(keras_model=model) +est.fit(data=df, + batch_size=64, + epochs=4, + feature_cols=['user', 'item'], label_cols=['label']) ``` - -## Get Started - -See [TensorFlow](../QuickStart/orca-tf-quickstart.md) and [PyTorch](../QuickStart/orca-pytorch-quickstart.md) quickstart for more details. - diff --git a/docs/readthedocs/source/doc/Ray/Overview/ray.md b/docs/readthedocs/source/doc/Orca/Overview/ray.md similarity index 93% rename from docs/readthedocs/source/doc/Ray/Overview/ray.md rename to docs/readthedocs/source/doc/Orca/Overview/ray.md index e7e20f87..7fde801c 100644 --- a/docs/readthedocs/source/doc/Ray/Overview/ray.md +++ b/docs/readthedocs/source/doc/Orca/Overview/ray.md @@ -2,9 +2,9 @@ --- -[Ray](https://github.com/ray-project/ray) is an open source distributed framework for emerging AI applications. -With the _**RayOnSpark**_ support packaged in [BigDL Orca](../../Orca/Overview/orca.md), -Users can seamlessly integrate Ray applications into the big data processing pipeline on the underlying Big Data cluster +[Ray](https://github.com/ray-project/ray) is an open source distributed framework for emerging AI applications. +With the _**RayOnSpark**_ support packaged in [BigDL Orca](../Overview/orca.md), +Users can seamlessly integrate Ray applications into the big data processing pipeline on the underlying Big Data cluster (such as [Hadoop/YARN](../../UserGuide/hadoop.md) or [K8s](../../UserGuide/k8s.md)). _**Note:** BigDL has been tested on Ray 1.9.2 and you are highly recommended to use this tested version._ @@ -12,8 +12,8 @@ _**Note:** BigDL has been tested on Ray 1.9.2 and you are highly recommended to ### **1. Install** -We recommend using [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to prepare the Python environment. -When installing bigdl-orca with pip, you can specify the extras key `[ray]` to install the additional dependencies +We recommend using [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) to prepare the Python environment. +When installing bigdl-orca with pip, you can specify the extras key `[ray]` to install the additional dependencies for running Ray (i.e. `ray==1.9.2`, `psutil`, `aiohttp==3.7.0`, `aioredis==1.1.0`, `setproctitle`, `hiredis==1.1.0`, `async-timeout==3.0.1`): ```bash @@ -23,7 +23,7 @@ conda activate py37 pip install bigdl-orca[ray] ``` -View [Python User Guide](../../UserGuide/python.html#install) and [Orca User Guide](../../Orca/Overview/orca.md) for more installation instructions. +View [Python User Guide](../../UserGuide/python.html#install) and [Orca User Guide](../Overview/orca.md) for more installation instructions. --- ### **2. Initialize** @@ -45,9 +45,9 @@ You can input the following RayOnSpark related arguments when you `init_orca_con - `extra_params`: The key value dict for extra options to launch ray. For example, `extra_params={"dashboard-port": "11281", "temp-dir": "/tmp/ray/"}`. - `include_webui`: Default is True for including web ui when starting ray. - `system_config`: The key value dict for overriding RayConfig defaults. Mainly for testing purposes. An example for system_config could be: `{"object_spilling_config":"{\"type\":\"filesystem\", \"params\":{\"directory_path\":\"/tmp/spill\"}}"}`. -- `num_ray_nodes`: The number of ray processes to start across the cluster. For Spark local mode, you don't need to specify this value. +- `num_ray_nodes`: The number of ray processes to start across the cluster. For Spark local mode, you don't need to specify this value. For Spark cluster mode, it is default to be the number of Spark executors. If spark.executor.instances can't be detected in your SparkContext, you need to explicitly specify this. It is recommended that num_ray_nodes is not larger than the number of Spark executors to make sure there are enough resources in your cluster. -- `ray_node_cpu_cores`: The number of available cores for each ray process. For Spark local mode, it is default to be the number of Spark local cores. +- `ray_node_cpu_cores`: The number of available cores for each ray process. For Spark local mode, it is default to be the number of Spark local cores. For Spark cluster mode, it is default to be the number of cores for each Spark executor. If spark.executor.cores or spark.cores.max can't be detected in your SparkContext, you need to explicitly specify this. It is recommended that ray_node_cpu_cores is not larger than the number of cores for each Spark executor to make sure there are enough resources in your cluster. By default, the Ray cluster would be launched using Spark barrier execution mode, you can turn it off via the configurations of `OrcaContext`: @@ -58,7 +58,7 @@ from bigdl.orca import OrcaContext OrcaContext.barrier_mode = False ``` -View [Orca Context](../../Orca/Overview/orca-context.md) for more details. +View [Orca Context](../Overview/orca-context.md) for more details. --- ### **3. Run** @@ -72,7 +72,7 @@ View [Orca Context](../../Orca/Overview/orca-context.md) for more details. class Counter(object): def __init__(self): self.n = 0 - + def increment(self): self.n += 1 return self.n @@ -82,11 +82,11 @@ View [Orca Context](../../Orca/Overview/orca-context.md) for more details. print(ray.get([c.increment.remote() for c in counters])) ``` -- You can retrieve the information of the Ray cluster via [`OrcaContext`](../../Orca/Overview/orca-context.md): +- You can retrieve the information of the Ray cluster via [`OrcaContext`](../Overview/orca-context.md): ```python from bigdl.orca import OrcaContext - + ray_ctx = OrcaContext.get_ray_context() address_info = ray_ctx.address_info # The dictionary information of the ray cluster, including node_ip_address, object_store_address, webui_url, etc. redis_address = ray_ctx.redis_address # The redis address of the ray cluster. @@ -96,7 +96,7 @@ View [Orca Context](../../Orca/Overview/orca-context.md) for more details. ```python from bigdl.orca import stop_orca_context - + stop_orca_context() ``` diff --git a/docs/readthedocs/source/doc/Orca/QuickStart/index.md b/docs/readthedocs/source/doc/Orca/QuickStart/index.md new file mode 100644 index 00000000..89751ddc --- /dev/null +++ b/docs/readthedocs/source/doc/Orca/QuickStart/index.md @@ -0,0 +1,44 @@ +# Orca Tutorial + + +- [**Orca TensorFlow 1.15 Quickstart**](./orca-tf-quickstart.html) + + > ![](../../../../image/colab_logo_32px.png)[Run in Google Colab](https://colab.research.google.com/github/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/tf_lenet_mnist.ipynb)  ![](../../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/tf_lenet_mnist.ipynb) + + In this guide we will describe how to scale out TensorFlow 1.15 programs using Orca in 4 simple steps. + +--------------------------- + +- [**Orca TensorFlow 2 Quickstart**](./orca-tf2keras-quickstart.html) + + > ![](../../../../image/colab_logo_32px.png)[Run in Google Colab](https://colab.research.google.com/github/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.ipynb)  ![](../../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.ipynb) + + In this guide we will describe how to to scale out TensorFlow 2 programs using Orca in 4 simple steps. + +--------------------------- + +- [**Orca Keras 2.3 Quickstart**](./orca-keras-quickstart.html) + + > ![](../../../../image/colab_logo_32px.png)[Run in Google Colab](https://colab.research.google.com/github/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/keras_lenet_mnist.ipynb)  ![](../../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/keras_lenet_mnist.ipynb) + + In this guide we will describe how to scale out Keras 2.3 programs using Orca in 4 simple steps. + +--------------------------- + + +- [**Orca PyTorch Quickstart**](./orca-pytorch-quickstart.html) + + > ![](../../../../image/colab_logo_32px.png)[Run in Google Colab](https://colab.research.google.com/github/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/pytorch_lenet_mnist.ipynb)  ![](../../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/blob/main/python/orca/colab-notebook/quickstart/pytorch_lenet_mnist.ipynb) + + In this guide we will describe how to scale out PyTorch programs using Orca in 4 simple steps. + +--------------------------- + +- [**Orca RayOnSpark Quickstart**](./ray-quickstart.html) + + > ![](../../../../image/colab_logo_32px.png)[Run in Google Colab](https://colab.research.google.com/github/intel-analytics/BigDL/blob/branch-2.0/python/orca/colab-notebook/quickstart/ray_parameter_server.ipynb)  ![](../../../../image/GitHub-Mark-32px.png)[View source on GitHub](https://github.com/intel-analytics/BigDL/blob/branch-2.0/python/orca/colab-notebook/quickstart/ray_parameter_server.ipynb) + + In this guide, we will describe how to use RayOnSpark to directly run Ray programs on Big Data clusters in 2 simple steps. + +--------------------------- + diff --git a/docs/readthedocs/source/doc/Ray/QuickStart/ray-quickstart.md b/docs/readthedocs/source/doc/Orca/QuickStart/ray-quickstart.md similarity index 93% rename from docs/readthedocs/source/doc/Ray/QuickStart/ray-quickstart.md rename to docs/readthedocs/source/doc/Orca/QuickStart/ray-quickstart.md index a485924d..b67d7cdf 100644 --- a/docs/readthedocs/source/doc/Ray/QuickStart/ray-quickstart.md +++ b/docs/readthedocs/source/doc/Orca/QuickStart/ray-quickstart.md @@ -33,7 +33,7 @@ elif cluster_mode == "yarn": # For Hadoop/YARN cluster sc = init_orca_context(cluster_mode="yarn", num_nodes=2, cores=2, memory="10g", driver_memory="10g", driver_cores=1, init_ray_on_spark=True) ``` -This is the only place where you need to specify local or distributed mode. See [here](./../../Ray/Overview/ray.md#initialize) for more RayOnSpark related arguments when you `init_orca_context`. +This is the only place where you need to specify local or distributed mode. See [here](./../Overview/ray.md#initialize) for more RayOnSpark related arguments when you `init_orca_context`. By default, the Ray cluster would be launched using Spark barrier execution mode, you can turn it off via the configurations of `OrcaContext`: @@ -43,7 +43,7 @@ from bigdl.orca import OrcaContext OrcaContext.barrier_mode = False ``` -View [Orca Context](./../../Orca/Overview/orca-context.md) for more details. +View [Orca Context](./../Overview/orca-context.md) for more details. **Note:** You should `export HADOOP_CONF_DIR=/path/to/hadoop/conf/dir` when running on Hadoop YARN cluster. View [Hadoop User Guide](./../../UserGuide/hadoop.md) for more details. @@ -76,10 +76,10 @@ dim = 10 class ParameterServer(object): def __init__(self, dim): self.parameters = np.zeros(dim) - + def get_parameters(self): return self.parameters - + def update_parameters(self, update): self.parameters += update diff --git a/docs/readthedocs/source/doc/Orca/index.rst b/docs/readthedocs/source/doc/Orca/index.rst new file mode 100644 index 00000000..f1c73fdd --- /dev/null +++ b/docs/readthedocs/source/doc/Orca/index.rst @@ -0,0 +1,63 @@ +BigDL-Orca +========================= + +Most AI projects start with a Python notebook running on a single laptop; however, one usually needs to go through a mountain of pains to scale it to handle larger data set in a distributed fashion. The **BigDL-Orca** (or **Orca** for short) library seamlessly scales out your single node Python notebook across large clusters (so as to process distributed Big Data). + + +------- + + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + Documents in these sections helps you get started quickly with Orca. + + +++ + :bdg-link:`Orca in 5 minutes <./Overview/orca.html>` | + :bdg-link:`Installation <./Overview/install.html>` + + .. grid-item-card:: + + **Key Features Guide** + ^^^ + + Each guide in this section provides you with in-depth information, concepts and knowledges about Orca key features. + + +++ + + :bdg-link:`Data <./Overview/data-parallel-processing.html>` | + :bdg-link:`Estimator <./Overview/distributed-training-inference.html>` | + :bdg-link:`RayOnSpark <./Overview/ray.html>` + + .. grid-item-card:: + + **Tutorials** + ^^^ + + Orca Tutorials and Examples. + + +++ + + :bdg-link:`Tutorials <./QuickStart/index.html>` + + .. grid-item-card:: + + **API Document** + ^^^ + + API Document provides detailed description of Orca APIs. + + +++ + + :bdg-link:`API Document <../PythonAPI/Orca/index.html>` + + +.. toctree:: + :hidden: + + BigDL-Orca Document diff --git a/docs/readthedocs/source/doc/PPML/Overview/azure_ppml.md b/docs/readthedocs/source/doc/PPML/Overview/azure_ppml.md index 9481a828..972ccc5a 100644 --- a/docs/readthedocs/source/doc/PPML/Overview/azure_ppml.md +++ b/docs/readthedocs/source/doc/PPML/Overview/azure_ppml.md @@ -26,7 +26,7 @@ az group create \ --location myLocation \ --output none ``` - + #### 2.2.2 Create Linux client with SGX support Create Linux VM through Azure [CLI](https://docs.microsoft.com/en-us/azure/developer/javascript/tutorial/nodejs-virtual-machine-vm/create-linux-virtual-machine-azure-cli)/[Portal](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-portal)/Powershell. For size of the VM, please choose DC-V3 Series VM with more than 4 vCPU cores. @@ -37,30 +37,32 @@ On `Subscribe` page, input your subscription, your Azure container registry, you * Go to your Azure container regsitry, check `Repostirories`, and find `intel_corporation/bigdl-ppml-trusted-big-data-ml-python-graphene` * Login to the created VM. Then login to your Azure container registry, pull BigDL PPML image using this command: -```bash -docker pull myContainerRegistry/intel_corporation/bigdl-ppml-trusted-big-data-ml-python-graphene -``` + ```bash + docker pull myContainerRegistry/intel_corporation/bigdl-ppml-trusted-big-data-ml-python-graphene + ``` * Start container of this image -```bash -#!/bin/bash -export LOCAL_IP=YOUR_LOCAL_IP -export DOCKER_IMAGE=intel_corporation/bigdl-ppml-trusted-big-data-ml-python-graphene + ```bash + #!/bin/bash -sudo docker run -itd \ - --privileged \ - --net=host \ - --cpuset-cpus="0-5" \ - --oom-kill-disable \ - --device=/dev/gsgx \ - --device=/dev/sgx/enclave \ - --device=/dev/sgx/provision \ - -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ - --name=spark-local \ - -e LOCAL_IP=$LOCAL_IP \ - -e SGX_MEM_SIZE=64G \ - $DOCKER_IMAGE bash -``` + export LOCAL_IP=YOUR_LOCAL_IP + export DOCKER_IMAGE=intel_corporation/bigdl-ppml-trusted-big-data-ml-python-graphene + + sudo docker run -itd \ + --privileged \ + --net=host \ + --cpuset-cpus="0-5" \ + --oom-kill-disable \ + --device=/dev/gsgx \ + --device=/dev/sgx/enclave \ + --device=/dev/sgx/provision \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + --name=spark-local \ + -e LOCAL_IP=$LOCAL_IP \ + -e SGX_MEM_SIZE=64G \ + $DOCKER_IMAGE bash + + ``` ### 2.3 Create AKS(Azure Kubernetes Services) or use existing AKs First, login to your client VM and enter your BigDL PPML container: @@ -89,34 +91,35 @@ You can check the information by running: /ppml/trusted-big-data-ml/azure/create-aks.sh --help ``` -## 2.4 Create Azure Data Lake Store Gen 2 -### 2.4.1 Create Data Lake Storage account or use an existing one. +### 2.4 Create Azure Data Lake Store Gen 2 +#### 2.4.1 Create Data Lake Storage account or use an existing one. The example command to create Data Lake store is as below: ```bash az dls account create --account myDataLakeAccount --location myLocation --resource-group myResourceGroup ``` * Create Container to put user data -Example command to create container -```bash -az storage fs create -n myFS --account-name myDataLakeAccount --auth-mode login -``` -* Create folder, upload file/folder -Example command to create folder: -```bash -az storage fs directory create -n myDirectory -f myFS --account-name myDataLakeAccount --auth-mode login -``` -Example command to upload file -```bash -az storage fs file upload -s "path/to/file" -p myDirectory/file -f myFS --account-name myDataLakeAccount --auth-mode login -``` -Example command to upload directory -```bash -az storage fs directory upload -f myFS --account-name myDataLakeAccount -s "path/to/directory" -d myDirectory --recursive -``` -### 2.4.2 Access data in Hadoop through ABFS(Azure Blob Filesystem) driver + Example command to create container + ```bash + az storage fs create -n myFS --account-name myDataLakeAccount --auth-mode login + ``` +* Create folder, upload file/folder + + Example command to create folder + ```bash + az storage fs directory create -n myDirectory -f myFS --account-name myDataLakeAccount --auth-mode login + ``` + Example command to upload file + ```bash + az storage fs file upload -s "path/to/file" -p myDirectory/file -f myFS --account-name myDataLakeAccount --auth-mode login + ``` + Example command to upload directory + ```bash + az storage fs directory upload -f myFS --account-name myDataLakeAccount -s "path/to/directory" -d myDirectory --recursive + ``` +#### 2.4.2 Access data in Hadoop through ABFS(Azure Blob Filesystem) driver You can access Data Lake Storage in Hadoop filesytem by such URI: ```abfs[s]://file_system@account_name.dfs.core.windows.net///``` -#### Authentication +##### Authentication The ABFS driver supports two forms of authentication so that the Hadoop application may securely access resources contained within a Data Lake Storage Gen2 capable account. - Shared Key: This permits users to access to ALL resources in the account. The key is encrypted and stored in Hadoop configuration. @@ -124,13 +127,13 @@ The ABFS driver supports two forms of authentication so that the Hadoop applicat By default, in our solution, we use shared key authentication. - Get Access key list of the storage account: -```bash -az storage account keys list -g MyResourceGroup -n myDataLakeAccount -``` + ```bash + az storage account keys list -g MyResourceGroup -n myDataLakeAccount + ``` Use one of the keys for authentication. -## 2.5 Create Azure Key Vault -### 2.5.1 Create or use an existing Azure Key Vault +### 2.5 Create Azure Key Vault +#### 2.5.1 Create or use an existing Azure Key Vault Example command to create key vault ```bash az keyvault create -n myKeyVault -g myResourceGroup -l location @@ -142,29 +145,30 @@ Take note of the following properties for use in the next section: * The name of your Azure key vault resource * The Azure tenant ID that the subscription belongs to -### 2.5.2 Set access policy for the client VM +#### 2.5.2 Set access policy for the client VM * Run such command to get the system identity: -```bash -az vm identity assign -g myResourceGroup -n myVM -``` -The output would be like this: -```bash -{ - "systemAssignedIdentity": "ff5505d6-8f72-4b99-af68-baff0fbd20f5", - "userAssignedIdentities": {} -} -``` -Take note of the systemAssignedIdentity of the client VM. + ```bash + az vm identity assign -g myResourceGroup -n myVM + ``` + The output would be like this: + ```bash + { + "systemAssignedIdentity": "ff5505d6-8f72-4b99-af68-baff0fbd20f5", + "userAssignedIdentities": {} + } + ``` + Take note of the systemAssignedIdentity of the client VM. * Set access policy for client VM -Example command: -```bash -az keyvault set-policy --name myKeyVault --object-id --secret-permissions all --key-permissions all --certificate-permissions all -``` -### 2.5.3 AKS access Key Vault -#### 2.5.3.1 Set access for AKS VM ScaleSet -##### a. Find your VM ScaleSet in your AKS, and assign system managed identity to VM ScaleSet. + Example command: + ```bash + az keyvault set-policy --name myKeyVault --object-id --secret-permissions all --key-permissions all --certificate-permissions all + ``` + +#### 2.5.3 AKS access Key Vault +##### 2.5.3.1 Set access for AKS VM ScaleSet +###### a. Find your VM ScaleSet in your AKS, and assign system managed identity to VM ScaleSet. ```bash az vm identity assign -g myResourceGroup -n myAKSVMSS ``` @@ -179,50 +183,53 @@ userAssignedIdentities: principalId: xxxxx ``` Take note of principalId of the first line as System Managed Identity of your VMSS. -##### b. Set access policy for AKS VM ScaleSet +###### b. Set access policy for AKS VM ScaleSet Example command: ```bash az keyvault set-policy --name myKeyVault --object-id --secret-permissions get --key-permissions all ``` -#### 2.5.3.2 Set access for AKS -##### a. Enable Azure Key Vault Provider for Secrets Store CSI Driver support +##### 2.5.3.2 Set access for AKS +###### a. Enable Azure Key Vault Provider for Secrets Store CSI Driver support Example command: ```bash az aks enable-addons --addons azure-keyvault-secrets-provider --name myAKSCluster --resource-group myResourceGroup ``` * Verify the Azure Key Vault Provider for Secrets Store CSI Driver installation -Example command: -```bash -kubectl get pods -n kube-system -l 'app in (secrets-store-csi-driver, secrets-store-provider-azure)' -``` -Be sure that a Secrets Store CSI Driver pod and an Azure Key Vault Provider pod are running on each node in your cluster's node pools. + + Example command: + ```bash + kubectl get pods -n kube-system -l 'app in (secrets-store-csi-driver, secrets-store-provider-azure)' + ``` + Be sure that a Secrets Store CSI Driver pod and an Azure Key Vault Provider pod are running on each node in your cluster's node pools. * Enable Azure Key Vault Provider for Secrets Store CSI Driver to track of secret update in key vault -```bash -az aks update -g myResourceGroup -n myAKSCluster --enable-secret-rotation -``` -#### b. Provide an identity to access the Azure Key Vault + ```bash + az aks update -g myResourceGroup -n myAKSCluster --enable-secret-rotation + ``` +###### b. Provide an identity to access the Azure Key Vault There are several ways to provide identity for Azure Key Vault Provider for Secrets Store CSI Driver to access Azure Key Vault: `An Azure Active Directory pod identity`, `user-assigned identity` or `system-assigned managed identity`. In our solution, we use user-assigned managed identity. * Enable managed identity in AKS -```bash -az aks update -g myResourceGroup -n myAKSCluster --enable-managed-identity -``` + ```bash + az aks update -g myResourceGroup -n myAKSCluster --enable-managed-identity + ``` * Get user-assigned managed identity that you created when you enabled a managed identity on your AKS cluster -Run: -```bash -az aks show -g myResourceGroup -n myAKSCluster --query addonProfiles.azureKeyvaultSecretsProvider.identity.clientId -o tsv -``` -The output would be like: -```bash -xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -``` -Take note of this output as your user-assigned managed identity of Azure KeyVault Secrets Provider + + Run: + ```bash + az aks show -g myResourceGroup -n myAKSCluster --query addonProfiles.azureKeyvaultSecretsProvider.identity.clientId -o tsv + ``` + The output would be like: + ```bash + xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + ``` + Take note of this output as your user-assigned managed identity of Azure KeyVault Secrets Provider * Grant your user-assigned managed identity permissions that enable it to read your key vault and view its contents -Example command: -```bash -az keyvault set-policy -n myKeyVault --key-permissions get --spn xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -az keyvault set-policy -n myKeyVault --secret-permissions get --spn xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -``` -#### c. Create a SecretProviderClass to access your Key Vault + + Example command: + ```bash + az keyvault set-policy -n myKeyVault --key-permissions get --spn xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + az keyvault set-policy -n myKeyVault --secret-permissions get --spn xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + ``` +###### c. Create a SecretProviderClass to access your Key Vault On your client docker container, edit `/ppml/trusted-big-data-ml/azure/secretProviderClass.yaml` file, modify `` to your user-assigned managed identity of Azure KeyVault Secrets Provider, and modify `` and `` to your real key vault name and tenant id. Then run: diff --git a/docs/readthedocs/source/doc/PPML/Overview/examples.rst b/docs/readthedocs/source/doc/PPML/Overview/examples.rst new file mode 100644 index 00000000..b53218f9 --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/Overview/examples.rst @@ -0,0 +1,8 @@ +Tutorials & Examples +===================================== + +* `A Hello World Example <../Overview/quicktour.html>`__ is a very simple exmaple for getting started. + +* `PPML e2e Example <../QuickStart/end-to-end.html>`__ introduces the end-to-end PPML workflow using SimpleQuery as an example. + +* You can also find Trusted Data Analysis, Trusted ML, Trusted DL and Trusted FL examples in `more examples `__. \ No newline at end of file diff --git a/docs/readthedocs/source/doc/PPML/Overview/intro.md b/docs/readthedocs/source/doc/PPML/Overview/intro.md new file mode 100644 index 00000000..99b2e02a --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/Overview/intro.md @@ -0,0 +1,35 @@ +# PPML Introduction + +## 1. What is BigDL PPML? + + + +--- + +Protecting data privacy and confidentiality is critical in a world where data is everywhere. In recent years, more and more countries have enacted data privacy legislation or are expected to pass comprehensive legislation to protect data privacy, the importance of privacy and data protection is increasingly recognized. + +To better protect sensitive data, it's necessary to ensure security for all dimensions of data lifecycle: data at rest, data in transit, and data in use. Data being transferred on a network is `in transit`, data in storage is `at rest`, and data being processed is `in use`. + +

    + data lifecycle +

    + +To protect data in transit, enterprises often choose to encrypt sensitive data prior to moving or use encrypted connections (HTTPS, SSL, TLS, FTPS, etc) to protect the contents of data in transit. For protecting data at rest, enterprises can simply encrypt sensitive files prior to storing them or choose to encrypt the storage drive itself. However, the third state, data in use has always been a weakly protected target. There are three emerging solutions seek to reduce the data-in-use attack surface: homomorphic encryption, multi-party computation, and confidential computing. + +Among these security technologies, [Confidential computing](https://www.intel.com/content/www/us/en/security/confidential-computing.html) protects data in use by performing computation in a hardware-based [Trusted Execution Environment (TEE)](https://en.wikipedia.org/wiki/Trusted_execution_environment). [Intel® SGX](https://www.intel.com/content/www/us/en/developer/tools/software-guard-extensions/overview.html) is Intel's Trusted Execution Environment (TEE), offering hardware-based memory encryption that isolates specific application code and data in memory. [Intel® TDX](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-trust-domain-extensions.html) is the next generation Intel's Trusted Execution Environment (TEE), introducing new, architectural elements to help deploy hardware-isolated, virtual machines (VMs) called trust domains (TDs). + +[PPML](https://bigdl.readthedocs.io/en/latest/doc/PPML/Overview/ppml.html) (Privacy Preserving Machine Learning) in [BigDL 2.0](https://github.com/intel-analytics/BigDL) provides a Trusted Cluster Environment for secure Big Data & AI applications, even on untrusted cloud environment. By combining Intel Software Guard Extensions (SGX) with several other security technologies (e.g., attestation, key management service, private set intersection, federated learning, homomorphic encryption, etc.), BigDL PPML ensures end-to-end security enabled for the entire distributed workflows, such as Apache Spark, Apache Flink, XGBoost, TensorFlow, PyTorch, etc. + + +## 2. Why BigDL PPML? +PPML allows organizations to explore powerful AI techniques while working to minimize the security risks associated with handling large amounts of sensitive data. PPML protects data at rest, in transit and in use: compute and memory protected by SGX Enclaves, storage (e.g., data and model) protected by encryption, network communication protected by remote attestation and Transport Layer Security (TLS), and optional Federated Learning support. + +

    + data lifecycle +

    + +With BigDL PPML, you can run trusted Big Data & AI applications +- **Trusted Spark SQL & Dataframe**: with the trusted Big Data analytics and ML/DL support, users can run standard Spark data analysis (such as Spark SQL, Dataframe, MLlib, etc.) in a secure and trusted fashion. +- **Trusted ML (Machine Learning)**: with the trusted Big Data analytics and ML/DL support, users can run distributed machine learning (such as MLlib, XGBoost) in a secure and trusted fashion. +- **Trusted DL (Deep Learning)**: with the trusted Big Data analytics and ML/DL support, users can run distributed deep learning (such as BigDL, Orca, Nano, DLlib) in a secure and trusted fashion. +- **Trusted FL (Federated Learning)**: with PSI (Private Set Intersection), Secured Aggregation and trusted federated learning support, users can build united model across different parties without compromising privacy, even if these parities have different datasets or features. diff --git a/docs/readthedocs/source/doc/PPML/Overview/misc.rst b/docs/readthedocs/source/doc/PPML/Overview/misc.rst new file mode 100644 index 00000000..471da90b --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/Overview/misc.rst @@ -0,0 +1,14 @@ +Advanced Topic +==================== + + +* `Privacy Preserving Machine Learning (PPML) User Guide `_ +* `Trusted Big Data Analytics and ML `_ +* `Trusted FL (Federated Learning) `_ +* `Secure Your Services <../QuickStart/secure_your_services.html>`_ +* `Building Linux Kernel from Source with SGX Enabled <../QuickStart/build_kernel_with_sgx.html>`_ +* `Deploy the Intel SGX Device Plugin for Kubernetes <../QuickStart/deploy_intel_sgx_device_plugin_for_kubernetes.html>`_ +* `Trusted Cluster Serving with Graphene on Kubernetes <../QuickStart/trusted-serving-on-k8s-guide.html>`_ +* `TPC-H with Trusted SparkSQL on Kubernetes <../QuickStart/tpc-h_with_sparksql_on_k8s.html>`_ +* `TPC-DS with Trusted SparkSQL on Kubernetes <../QuickStart/tpc-ds_with_sparksql_on_k8s.html>`_ +* `Privacy Preserving Machine Learning (PPML) on Azure User Guide `_ \ No newline at end of file diff --git a/docs/readthedocs/source/doc/PPML/Overview/ppml.md b/docs/readthedocs/source/doc/PPML/Overview/ppml.md index 5c41141d..df61d00a 100644 --- a/docs/readthedocs/source/doc/PPML/Overview/ppml.md +++ b/docs/readthedocs/source/doc/PPML/Overview/ppml.md @@ -230,29 +230,30 @@ Follow the guide below to run Spark on Kubernetes manually. Alternatively, you c 1. Enter `BigDL/ppml/trusted-big-data-ml/python/docker-graphene` dir. Refer to the previous section about [preparing data, keys and passwords](#2221-start-ppml-container). Then run the following commands to generate your enclave key and add it to your Kubernetes cluster as a secret. -```bash -kubectl apply -f keys/keys.yaml -kubectl apply -f password/password.yaml -cd kubernetes -bash enclave-key-to-secret.sh -``` + ```bash + kubectl apply -f keys/keys.yaml + kubectl apply -f password/password.yaml + cd kubernetes + bash enclave-key-to-secret.sh + ``` 2. Create the [RBAC(Role-based access control)](https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac) : -```bash -kubectl create serviceaccount spark -kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default -``` + ```bash + kubectl create serviceaccount spark + kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default + ``` 3. Generate K8s config file, modify `YOUR_DIR` to the location you want to store the config: -```bash -kubectl config view --flatten --minify > /YOUR_DIR/kubeconfig -``` + ```bash + kubectl config view --flatten --minify > /YOUR_DIR/kubeconfig + ``` + 4. Create K8s secret, the secret created `YOUR_SECRET` should be the same as the password you specified in step 1: -```bash -kubectl create secret generic spark-secret --from-literal secret=YOUR_SECRET -``` + ```bash + kubectl create secret generic spark-secret --from-literal secret=YOUR_SECRET + ``` ##### 2.2.3.2 Start the client container @@ -309,75 +310,75 @@ sudo docker run -itd \ 1. Run `docker exec -it spark-local-k8s-client bash` to enter the container. Then run the following command to init the Spark local K8s client. -```bash -./init.sh -``` + ```bash + ./init.sh + ``` 2. We assume you have a working Network File System (NFS) configured for your Kubernetes cluster. Configure the `nfsvolumeclaim` on the last line to the name of the Persistent Volume Claim (PVC) of your NFS. Please prepare the following and put them in your NFS directory: -- The data (in a directory called `data`) -- The kubeconfig file. + - The data (in a directory called `data`) + - The kubeconfig file. 3. Run the following command to start Spark-Pi example. When the application runs in `cluster` mode, you can run ` kubectl get pod ` to get the name and status of your K8s pod(e.g., driver-xxxx). Then you can run ` kubectl logs -f driver-xxxx ` to get the output of your application. -```bash -#!/bin/bash -secure_password=`openssl rsautl -inkey /ppml/trusted-big-data-ml/work/password/key.txt -decrypt &1 | tee spark-pi-sgx-$SPARK_MODE.log -``` + ```bash + #!/bin/bash + secure_password=`openssl rsautl -inkey /ppml/trusted-big-data-ml/work/password/key.txt -decrypt &1 | tee spark-pi-sgx-$SPARK_MODE.log + ``` You can run your own Spark application after changing `--class` and jar path. diff --git a/docs/readthedocs/source/doc/PPML/Overview/quicktour.md b/docs/readthedocs/source/doc/PPML/Overview/quicktour.md new file mode 100644 index 00000000..5fa88411 --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/Overview/quicktour.md @@ -0,0 +1,92 @@ +# A Hello World Example + + +In this section, you can get started with running a simple native python HelloWorld program and a simple native Spark Pi program locally in a BigDL PPML client container to get an initial understanding of the usage of ppml. + + + +## a. Prepare Keys + +* generate ssl_key + + Download scripts from [here](https://github.com/intel-analytics/BigDL). + + ``` + cd BigDL/ppml/ + sudo bash scripts/generate-keys.sh + ``` + This script will generate keys under keys/ folder + +* generate enclave-key.pem + + ``` + openssl genrsa -3 -out enclave-key.pem 3072 + ``` + This script generates a file enclave-key.pem which is used to sign image. + + +## b. Start the BigDL PPML client container + +``` +#!/bin/bash + +# ENCLAVE_KEY_PATH means the absolute path to the "enclave-key.pem" in step a +# KEYS_PATH means the absolute path to the keys folder in step a +# LOCAL_IP means your local IP address. +export ENCLAVE_KEY_PATH=YOUR_LOCAL_ENCLAVE_KEY_PATH +export KEYS_PATH=YOUR_LOCAL_KEYS_PATH +export LOCAL_IP=YOUR_LOCAL_IP +export DOCKER_IMAGE=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:devel + +sudo docker pull $DOCKER_IMAGE + +sudo docker run -itd \ + --privileged \ + --net=host \ + --cpuset-cpus="0-5" \ + --oom-kill-disable \ + --device=/dev/gsgx \ + --device=/dev/sgx/enclave \ + --device=/dev/sgx/provision \ + -v $ENCLAVE_KEY_PATH:/graphene/Pal/src/host/Linux-SGX/signer/enclave-key.pem \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + -v $KEYS_PATH:/ppml/trusted-big-data-ml/work/keys \ + --name=bigdl-ppml-client-local \ + -e LOCAL_IP=$LOCAL_IP \ + -e SGX_MEM_SIZE=64G \ + $DOCKER_IMAGE bash +``` + +## c. Run Python HelloWorld in BigDL PPML Client Container + +Run the [script](https://github.com/intel-analytics/BigDL/blob/main/ppml/trusted-big-data-ml/python/docker-graphene/start-scripts/start-python-helloworld-sgx.sh) to run trusted [Python HelloWorld](https://github.com/intel-analytics/BigDL/blob/main/ppml/trusted-big-data-ml/python/docker-graphene/examples/helloworld.py) in BigDL PPML client container: +``` +sudo docker exec -it bigdl-ppml-client-local bash work/start-scripts/start-python-helloworld-sgx.sh +``` +Check the log: +``` +sudo docker exec -it bigdl-ppml-client-local cat /ppml/trusted-big-data-ml/test-helloworld-sgx.log | egrep "Hello World" +``` +The result should look something like this: +> Hello World + + +## d. Run Spark Pi in BigDL PPML Client Container + +Run the [script](https://github.com/intel-analytics/BigDL/blob/main/ppml/trusted-big-data-ml/python/docker-graphene/start-scripts/start-spark-local-pi-sgx.sh) to run trusted [Spark Pi](https://github.com/apache/spark/blob/v3.1.2/examples/src/main/python/pi.py) in BigDL PPML client container: + +```bash +sudo docker exec -it bigdl-ppml-client-local bash work/start-scripts/start-spark-local-pi-sgx.sh +``` + +Check the log: + +```bash +sudo docker exec -it bigdl-ppml-client-local cat /ppml/trusted-big-data-ml/test-pi-sgx.log | egrep "roughly" +``` + +The result should look something like this: + +> Pi is roughly 3.146760 + +
    \ No newline at end of file diff --git a/docs/readthedocs/source/doc/PPML/Overview/userguide.md b/docs/readthedocs/source/doc/PPML/Overview/userguide.md new file mode 100644 index 00000000..e109756e --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/Overview/userguide.md @@ -0,0 +1,504 @@ +## Develop your own Big Data & AI applications with BigDL PPML + +First you need to create a `PPMLContext`, which wraps `SparkSession` and provides methods to read encrypted data file into plain-text RDD/DataFrame and write DataFrame to encrypted data file. Then you can read & write data through `PPMLContext`. + +If you are familiar with Spark, you may find that the usage of `PPMLConext` is very similar to Spark. + +### 1. Create PPMLContext + +- create a PPMLContext with `appName` + + This is the simplest way to create a `PPMLContext`. When you don't need to read/write encrypted files, you can use this way to create a `PPMLContext`. + +
    + scala + + ```scala + import com.intel.analytics.bigdl.ppml.PPMLContext + + val sc = PPMLContext.initPPMLContext("MyApp") + ``` + +
    + +
    + python + + ```python + from bigdl.ppml.ppml_context import * + + sc = PPMLContext("MyApp") + ``` + +
    + + If you want to read/write encrypted files, then you need to provide more information. + +- create a PPMLContext with `appName` & `ppmlArgs` + + `ppmlArgs` is ppml arguments in a Map, `ppmlArgs` varies according to the kind of Key Management Service (KMS) you are using. Key Management Service (KMS) is used to generate `primaryKey` and `dataKey` to encrypt/decrypt data. We provide 3 types of KMS ——SimpleKeyManagementService, EHSMKeyManagementService, AzureKeyManagementService. + + Refer to [KMS Utils](https://github.com/intel-analytics/BigDL/blob/main/ppml/services/kms-utils/docker/README.md) to use KMS to generate `primaryKey` and `dataKey`, then you are ready to create **PPMLContext** with `ppmlArgs`. + + - For `SimpleKeyManagementService`: + +
    + scala + + ```scala + import com.intel.analytics.bigdl.ppml.PPMLContext + + val ppmlArgs: Map[String, String] = Map( + "spark.bigdl.kms.type" -> "SimpleKeyManagementService", + "spark.bigdl.kms.simple.id" -> "your_app_id", + "spark.bigdl.kms.simple.key" -> "your_app_key", + "spark.bigdl.kms.key.primary" -> "/your/primary/key/path/primaryKey", + "spark.bigdl.kms.key.data" -> "/your/data/key/path/dataKey" + ) + + val sc = PPMLContext.initPPMLContext("MyApp", ppmlArgs) + ``` + +
    + + +
    + python + + ```python + from bigdl.ppml.ppml_context import * + + ppml_args = {"kms_type": "SimpleKeyManagementService", + "simple_app_id": "your_app_id", + "simple_app_key": "your_app_key", + "primary_key_path": "/your/primary/key/path/primaryKey", + "data_key_path": "/your/data/key/path/dataKey" + } + + sc = PPMLContext("MyApp", ppml_args) + ``` + +
    + + - For `EHSMKeyManagementService`: + +
    + scala + + ```scala + import com.intel.analytics.bigdl.ppml.PPMLContext + + val ppmlArgs: Map[String, String] = Map( + "spark.bigdl.kms.type" -> "EHSMKeyManagementService", + "spark.bigdl.kms.ehs.ip" -> "your_server_ip", + "spark.bigdl.kms.ehs.port" -> "your_server_port", + "spark.bigdl.kms.ehs.id" -> "your_app_id", + "spark.bigdl.kms.ehs.key" -> "your_app_key", + "spark.bigdl.kms.key.primary" -> "/your/primary/key/path/primaryKey", + "spark.bigdl.kms.key.data" -> "/your/data/key/path/dataKey" + ) + + val sc = PPMLContext.initPPMLContext("MyApp", ppmlArgs) + ``` + +
    + +
    + python + + ```python + from bigdl.ppml.ppml_context import * + + ppml_args = {"kms_type": "EHSMKeyManagementService", + "kms_server_ip": "your_server_ip", + "kms_server_port": "your_server_port" + "ehsm_app_id": "your_app_id", + "ehsm_app_key": "your_app_key", + "primary_key_path": "/your/primary/key/path/primaryKey", + "data_key_path": "/your/data/key/path/dataKey" + } + + sc = PPMLContext("MyApp", ppml_args) + ``` + +
    + + - For `AzureKeyManagementService` + + + the parameter `clientId` is not necessary, you don't have to provide this parameter. + +
    + scala + + ```scala + import com.intel.analytics.bigdl.ppml.PPMLContext + + val ppmlArgs: Map[String, String] = Map( + "spark.bigdl.kms.type" -> "AzureKeyManagementService", + "spark.bigdl.kms.azure.vault" -> "key_vault_name", + "spark.bigdl.kms.azure.clientId" -> "client_id", + "spark.bigdl.kms.key.primary" -> "/your/primary/key/path/primaryKey", + "spark.bigdl.kms.key.data" -> "/your/data/key/path/dataKey" + ) + + val sc = PPMLContext.initPPMLContext("MyApp", ppmlArgs) + ``` + +
    + +
    + python + + ```python + from bigdl.ppml.ppml_context import * + + ppml_args = {"kms_type": "AzureKeyManagementService", + "azure_vault": "your_azure_vault", + "azure_client_id": "your_azure_client_id", + "primary_key_path": "/your/primary/key/path/primaryKey", + "data_key_path": "/your/data/key/path/dataKey" + } + + sc = PPMLContext("MyApp", ppml_args) + ``` + +
    + +- create a PPMLContext with `sparkConf` & `appName` & `ppmlArgs` + + If you need to set Spark configurations, you can provide a `SparkConf` with Spark configurations to create a `PPMLContext`. + +
    + scala + + ```scala + import com.intel.analytics.bigdl.ppml.PPMLContext + import org.apache.spark.SparkConf + + val ppmlArgs: Map[String, String] = Map( + "spark.bigdl.kms.type" -> "SimpleKeyManagementService", + "spark.bigdl.kms.simple.id" -> "your_app_id", + "spark.bigdl.kms.simple.key" -> "your_app_key", + "spark.bigdl.kms.key.primary" -> "/your/primary/key/path/primaryKey", + "spark.bigdl.kms.key.data" -> "/your/data/key/path/dataKey" + ) + + val conf: SparkConf = new SparkConf().setMaster("local[4]") + + val sc = PPMLContext.initPPMLContext(conf, "MyApp", ppmlArgs) + ``` + +
    + +
    + python + + ```python + from bigdl.ppml.ppml_context import * + from pyspark import SparkConf + + ppml_args = {"kms_type": "SimpleKeyManagementService", + "simple_app_id": "your_app_id", + "simple_app_key": "your_app_key", + "primary_key_path": "/your/primary/key/path/primaryKey", + "data_key_path": "/your/data/key/path/dataKey" + } + + conf = SparkConf() + conf.setMaster("local[4]") + + sc = PPMLContext("MyApp", ppml_args, conf) + ``` + +
    + +### 2. Read and Write Files + +To read/write data, you should set the `CryptoMode`: + +- `plain_text`: no encryption +- `AES/CBC/PKCS5Padding`: for CSV, JSON and text file +- `AES_GCM_V1`: for PARQUET only +- `AES_GCM_CTR_V1`: for PARQUET only + +To write data, you should set the `write` mode: + +- `overwrite`: Overwrite existing data with the content of dataframe. +- `append`: Append content of the dataframe to existing data or table. +- `ignore`: Ignore current write operation if data / table already exists without any error. +- `error`: Throw an exception if data or table already exists. +- `errorifexists`: Throw an exception if data or table already exists. + +
    + scala + +```scala +import com.intel.analytics.bigdl.ppml.crypto.{AES_CBC_PKCS5PADDING, PLAIN_TEXT} + +// read data +val df = sc.read(cryptoMode = PLAIN_TEXT) + ... + +// write data +sc.write(dataFrame = df, cryptoMode = AES_CBC_PKCS5PADDING) +.mode("overwrite") +... +``` + +
    + +
    + python + +```python +from bigdl.ppml.ppml_context import * + +# read data +df = sc.read(crypto_mode = CryptoMode.PLAIN_TEXT) + ... + +# write data +sc.write(dataframe = df, crypto_mode = CryptoMode.AES_CBC_PKCS5PADDING) +.mode("overwrite") +... +``` + +
    + +
    expand to see the examples of reading/writing CSV, PARQUET, JSON and text file + +The following examples use `sc` to represent a initialized `PPMLContext` + +**read/write CSV file** + +
    + scala + +```scala +import com.intel.analytics.bigdl.ppml.PPMLContext +import com.intel.analytics.bigdl.ppml.crypto.{AES_CBC_PKCS5PADDING, PLAIN_TEXT} + +// read a plain csv file and return a DataFrame +val plainCsvPath = "/plain/csv/path" +val df1 = sc.read(cryptoMode = PLAIN_TEXT).option("header", "true").csv(plainCsvPath) + +// write a DataFrame as a plain csv file +val plainOutputPath = "/plain/output/path" +sc.write(df1, PLAIN_TEXT) +.mode("overwrite") +.option("header", "true") +.csv(plainOutputPath) + +// read a encrypted csv file and return a DataFrame +val encryptedCsvPath = "/encrypted/csv/path" +val df2 = sc.read(cryptoMode = AES_CBC_PKCS5PADDING).option("header", "true").csv(encryptedCsvPath) + +// write a DataFrame as a encrypted csv file +val encryptedOutputPath = "/encrypted/output/path" +sc.write(df2, AES_CBC_PKCS5PADDING) +.mode("overwrite") +.option("header", "true") +.csv(encryptedOutputPath) +``` + +
    + +
    + python + +```python +# import +from bigdl.ppml.ppml_context import * + +# read a plain csv file and return a DataFrame +plain_csv_path = "/plain/csv/path" +df1 = sc.read(CryptoMode.PLAIN_TEXT).option("header", "true").csv(plain_csv_path) + +# write a DataFrame as a plain csv file +plain_output_path = "/plain/output/path" +sc.write(df1, CryptoMode.PLAIN_TEXT) +.mode('overwrite') +.option("header", True) +.csv(plain_output_path) + +# read a encrypted csv file and return a DataFrame +encrypted_csv_path = "/encrypted/csv/path" +df2 = sc.read(CryptoMode.AES_CBC_PKCS5PADDING).option("header", "true").csv(encrypted_csv_path) + +# write a DataFrame as a encrypted csv file +encrypted_output_path = "/encrypted/output/path" +sc.write(df2, CryptoMode.AES_CBC_PKCS5PADDING) +.mode('overwrite') +.option("header", True) +.csv(encrypted_output_path) +``` + +
    + +**read/write PARQUET file** + +
    + scala + +```scala +import com.intel.analytics.bigdl.ppml.PPMLContext +import com.intel.analytics.bigdl.ppml.crypto.{AES_GCM_CTR_V1, PLAIN_TEXT} + +// read a plain parquet file and return a DataFrame +val plainParquetPath = "/plain/parquet/path" +val df1 = sc.read(PLAIN_TEXT).parquet(plainParquetPath) + +// write a DataFrame as a plain parquet file +plainOutputPath = "/plain/output/path" +sc.write(df1, PLAIN_TEXT) +.mode("overwrite") +.parquet(plainOutputPath) + +// read a encrypted parquet file and return a DataFrame +val encryptedParquetPath = "/encrypted/parquet/path" +val df2 = sc.read(AES_GCM_CTR_V1).parquet(encryptedParquetPath) + +// write a DataFrame as a encrypted parquet file +val encryptedOutputPath = "/encrypted/output/path" +sc.write(df2, AES_GCM_CTR_V1) +.mode("overwrite") +.parquet(encryptedOutputPath) +``` + +
    + + +
    + python + +```python +# import +from bigdl.ppml.ppml_context import * + +# read a plain parquet file and return a DataFrame +plain_parquet_path = "/plain/parquet/path" +df1 = sc.read(CryptoMode.PLAIN_TEXT).parquet(plain_parquet_path) + +# write a DataFrame as a plain parquet file +plain_output_path = "/plain/output/path" +sc.write(df1, CryptoMode.PLAIN_TEXT) +.mode('overwrite') +.parquet(plain_output_path) + +# read a encrypted parquet file and return a DataFrame +encrypted_parquet_path = "/encrypted/parquet/path" +df2 = sc.read(CryptoMode.AES_GCM_CTR_V1).parquet(encrypted_parquet_path) + +# write a DataFrame as a encrypted parquet file +encrypted_output_path = "/encrypted/output/path" +sc.write(df2, CryptoMode.AES_GCM_CTR_V1) +.mode('overwrite') +.parquet(encrypted_output_path) +``` + +
    + +**read/write JSON file** + +
    + scala + +```scala +import com.intel.analytics.bigdl.ppml.PPMLContext +import com.intel.analytics.bigdl.ppml.crypto.{AES_CBC_PKCS5PADDING, PLAIN_TEXT} + +// read a plain json file and return a DataFrame +val plainJsonPath = "/plain/json/path" +val df1 = sc.read(PLAIN_TEXT).json(plainJsonPath) + +// write a DataFrame as a plain json file +val plainOutputPath = "/plain/output/path" +sc.write(df1, PLAIN_TEXT) +.mode("overwrite") +.json(plainOutputPath) + +// read a encrypted json file and return a DataFrame +val encryptedJsonPath = "/encrypted/parquet/path" +val df2 = sc.read(AES_CBC_PKCS5PADDING).json(encryptedJsonPath) + +// write a DataFrame as a encrypted parquet file +val encryptedOutputPath = "/encrypted/output/path" +sc.write(df2, AES_CBC_PKCS5PADDING) +.mode("overwrite") +.json(encryptedOutputPath) +``` + +
    + +
    + python + +```python +# import +from bigdl.ppml.ppml_context import * + +# read a plain json file and return a DataFrame +plain_json_path = "/plain/json/path" +df1 = sc.read(CryptoMode.PLAIN_TEXT).json(plain_json_path) + +# write a DataFrame as a plain json file +plain_output_path = "/plain/output/path" +sc.write(df1, CryptoMode.PLAIN_TEXT) +.mode('overwrite') +.json(plain_output_path) + +# read a encrypted json file and return a DataFrame +encrypted_json_path = "/encrypted/parquet/path" +df2 = sc.read(CryptoMode.AES_CBC_PKCS5PADDING).json(encrypted_json_path) + +# write a DataFrame as a encrypted parquet file +encrypted_output_path = "/encrypted/output/path" +sc.write(df2, CryptoMode.AES_CBC_PKCS5PADDING) +.mode('overwrite') +.json(encrypted_output_path) +``` + +
    + +**read textfile** + +
    + scala + +```scala +import com.intel.analytics.bigdl.ppml.PPMLContext +import com.intel.analytics.bigdl.ppml.crypto.{AES_CBC_PKCS5PADDING, PLAIN_TEXT} + +// read from a plain csv file and return a RDD +val plainCsvPath = "/plain/csv/path" +val rdd1 = sc.textfile(plainCsvPath) // the default cryptoMode is PLAIN_TEXT + +// read from a encrypted csv file and return a RDD +val encryptedCsvPath = "/encrypted/csv/path" +val rdd2 = sc.textfile(path=encryptedCsvPath, cryptoMode=AES_CBC_PKCS5PADDING) +``` + +
    + +
    + python + +```python +# import +from bigdl.ppml.ppml_context import * + +# read from a plain csv file and return a RDD +plain_csv_path = "/plain/csv/path" +rdd1 = sc.textfile(plain_csv_path) # the default crypto_mode is "plain_text" + +# read from a encrypted csv file and return a RDD +encrypted_csv_path = "/encrypted/csv/path" +rdd2 = sc.textfile(path=encrypted_csv_path, crypto_mode=CryptoMode.AES_CBC_PKCS5PADDING) +``` + +
    + +
    + +More usage with `PPMLContext` Python API, please refer to [PPMLContext Python API](https://github.com/intel-analytics/BigDL/blob/main/python/ppml/src/bigdl/ppml/README.md). diff --git a/docs/readthedocs/source/doc/PPML/QuickStart/end-to-end.md b/docs/readthedocs/source/doc/PPML/QuickStart/end-to-end.md new file mode 100644 index 00000000..3acbd242 --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/QuickStart/end-to-end.md @@ -0,0 +1,175 @@ +# PPML End-to-End Workflow Example + +## E2E Architecture Overview + +In this section we take SimpleQuery as an example to go through the entire BigDL PPML end-to-end workflow. SimpleQuery is simple example to query developers between the ages of 20 and 40 from people.csv. + + +

    + data lifecycle +

    + + + +--- + +## Step 0. Preparation your environment +To secure your Big Data & AI applications in BigDL PPML manner, you should prepare your environment first, including K8s cluster setup, K8s-SGX plugin setup, key/password preparation, key management service (KMS) and attestation service (AS) setup, BigDL PPML client container preparation. **Please follow the detailed steps in** [Prepare Environment](./docs/prepare_environment.md). + + +## Step 1. Encrypt and Upload Data +Encrypt the input data of your Big Data & AI applications (here we use SimpleQuery) and then upload encrypted data to the nfs server. More details in [Encrypt Your Data](./services/kms-utils/docker/README.md#3-enroll-generate-key-encrypt-and-decrypt). + +1. Generate the input data `people.csv` for SimpleQuery application +you can use [generate_people_csv.py](https://github.com/analytics-zoo/ppml-e2e-examples/blob/main/spark-encrypt-io/generate_people_csv.py). The usage command of the script is `python generate_people.py `. + +2. Encrypt `people.csv` + ``` + docker exec -i $KMSUTIL_CONTAINER_NAME bash -c "bash /home/entrypoint.sh encrypt $appid $apikey $input_file_path" + ``` +## Step 2. Build Big Data & AI applications +To build your own Big Data & AI applications, refer to [develop your own Big Data & AI applications with BigDL PPML](#4-develop-your-own-big-data--ai-applications-with-bigdl-ppml). The code of SimpleQuery is in [here](https://github.com/intel-analytics/BigDL/blob/main/scala/ppml/src/main/scala/com/intel/analytics/bigdl/ppml/examples/SimpleQuerySparkExample.scala), it is already built into bigdl-ppml-spark_3.1.2-2.1.0-SNAPSHOT.jar, and the jar is put into PPML image. + +## Step 3. Attestation + +To enable attestation, you should have a running Attestation Service (EHSM-KMS here for example) in your environment. (You can start a KMS refering to [this link](https://github.com/intel-analytics/BigDL/tree/main/ppml/services/kms-utils/docker)). Configure your KMS app_id and app_key with `kubectl`, and then configure KMS settings in `spark-driver-template.yaml` and `spark-executor-template.yaml` in the container. +``` bash +kubectl create secret generic kms-secret --from-literal=app_id=your-kms-app-id --from-literal=app_key=your-kms-app-key +``` +Configure `spark-driver-template.yaml` for example. (`spark-executor-template.yaml` is similar) +``` yaml +apiVersion: v1 +kind: Pod +spec: + containers: + - name: spark-driver + securityContext: + privileged: true + env: + - name: ATTESTATION + value: true + - name: ATTESTATION_URL + value: your_attestation_url + - name: ATTESTATION_ID + valueFrom: + secretKeyRef: + name: kms-secret + key: app_id + - name: ATTESTATION_KEY + valueFrom: + secretKeyRef: + name: kms-secret + key: app_key +... +``` +You should get `Attestation Success!` in logs after you [submit a PPML job](#step-4-submit-job) if the quote generated with user report is verified successfully by Attestation Service, or you will get `Attestation Fail! Application killed!` and the job will be stopped. + +## Step 4. Submit Job +When the Big Data & AI application and its input data is prepared, you are ready to submit BigDL PPML jobs. You need to choose the deploy mode and the way to submit job first. + +* **There are 4 modes to submit job**: + + 1. **local mode**: run jobs locally without connecting to cluster. It is exactly same as using spark-submit to run your application: `$SPARK_HOME/bin/spark-submit --class "SimpleApp" --master local[4] target.jar`, driver and executors are not protected by SGX. +

    + +

    + + + 2. **local SGX mode**: run jobs locally with SGX guarded. As the picture shows, the client JVM is running in a SGX Enclave so that driver and executors can be protected. +

    + +

    + + + 3. **client SGX mode**: run jobs in k8s client mode with SGX guarded. As we know, in K8s client mode, the driver is deployed locally as an external client to the cluster. With **client SGX mode**, the executors running in K8S cluster are protected by SGX, the driver running in client is also protected by SGX. +

    + +

    + + + 4. **cluster SGX mode**: run jobs in k8s cluster mode with SGX guarded. As we know, in K8s cluster mode, the driver is deployed on the k8s worker nodes like executors. With **cluster SGX mode**, the driver and executors running in K8S cluster are protected by SGX. +

    + +

    + + +* **There are two options to submit PPML jobs**: + * use [PPML CLI](./docs/submit_job.md#ppml-cli) to submit jobs manually + * use [helm chart](./docs/submit_job.md#helm-chart) to submit jobs automatically + +Here we use **k8s client mode** and **PPML CLI** to run SimpleQuery. Check other modes, please see [PPML CLI Usage Examples](./docs/submit_job.md#usage-examples). Alternatively, you can also use Helm to submit jobs automatically, see the details in [Helm Chart Usage](./docs/submit_job.md#helm-chart). + +
    expand to see details of submitting SimpleQuery + + 1. enter the ppml container + ``` + docker exec -it bigdl-ppml-client-k8s bash + ``` + 2. run simplequery on k8s client mode + ``` + #!/bin/bash + export secure_password=`openssl rsautl -inkey /ppml/trusted-big-data-ml/work/password/key.txt -decrypt +
    + +## Step 5. Decrypt and Read Result +When the job is done, you can decrypt and read result of the job. More details in [Decrypt Job Result](./services/kms-utils/docker/README.md#3-enroll-generate-key-encrypt-and-decrypt). + + ``` + docker exec -i $KMSUTIL_CONTAINER_NAME bash -c "bash /home/entrypoint.sh decrypt $appid $apikey $input_path" + ``` + +## Video Demo + + \ No newline at end of file diff --git a/docs/readthedocs/source/doc/PPML/QuickStart/tpc-ds_with_sparksql_on_k8s.md b/docs/readthedocs/source/doc/PPML/QuickStart/tpc-ds_with_sparksql_on_k8s.md index b043703b..e70f75f2 100644 --- a/docs/readthedocs/source/doc/PPML/QuickStart/tpc-ds_with_sparksql_on_k8s.md +++ b/docs/readthedocs/source/doc/PPML/QuickStart/tpc-ds_with_sparksql_on_k8s.md @@ -10,205 +10,205 @@ 1. Download and compile tpc-ds -```bash -git clone --recursive https://github.com/intel-analytics/zoo-tutorials.git -cd /path/to/zoo-tutorials -git clone https://github.com/databricks/tpcds-kit.git -cd tpcds-kit/tools -make OS=LINUX -``` + ```bash + git clone --recursive https://github.com/intel-analytics/zoo-tutorials.git + cd /path/to/zoo-tutorials + git clone https://github.com/databricks/tpcds-kit.git + cd tpcds-kit/tools + make OS=LINUX + ``` 2. Generate data -```bash -cd /path/to/zoo-tutorials -cd tpcds-spark/spark-sql-perf -sbt "test:runMain com.databricks.spark.sql.perf.tpcds.GenTPCDSData -d -s -l -f parquet" -``` + ```bash + cd /path/to/zoo-tutorials + cd tpcds-spark/spark-sql-perf + sbt "test:runMain com.databricks.spark.sql.perf.tpcds.GenTPCDSData -d -s -l -f parquet" + ``` -`dsdgenDir` is the path of `tpcds-kit/tools`, `scaleFactor` is the size of the data, for example `-s 1` will generate 1G data, `dataDir` is the path to store generated data. + `dsdgenDir` is the path of `tpcds-kit/tools`, `scaleFactor` is the size of the data, for example `-s 1` will generate 1G data, `dataDir` is the path to store generated data. ### Deploy PPML TPC-DS on Kubernetes 1. Compile Kit -```bash -cd zoo-tutorials/tpcds-spark -sbt package -``` + ```bash + cd zoo-tutorials/tpcds-spark + sbt package + ``` 2. Create external tables -```bash -$SPARK_HOME/bin/spark-submit \ - --class "createTables" \ - --master \ - --driver-memory 20G \ - --executor-cores \ - --total-executor-cores \ - --executor-memory 20G \ - --jars spark-sql-perf/target/scala-2.12/spark-sql-perf_2.12-0.5.1-SNAPSHOT.jar \ - target/scala-2.12/tpcds-benchmark_2.12-0.1.jar -``` + ```bash + $SPARK_HOME/bin/spark-submit \ + --class "createTables" \ + --master \ + --driver-memory 20G \ + --executor-cores \ + --total-executor-cores \ + --executor-memory 20G \ + --jars spark-sql-perf/target/scala-2.12/spark-sql-perf_2.12-0.5.1-SNAPSHOT.jar \ + target/scala-2.12/tpcds-benchmark_2.12-0.1.jar + ``` 3. Pull docker image -```bash -sudo docker pull intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT -``` + ```bash + sudo docker pull intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT + ``` 4. Prepare SGX keys (following instructions [here](https://github.com/intel-analytics/BigDL/tree/main/ppml/trusted-big-data-ml/python/docker-graphene#11-prepare-the-keyspassworddataenclave-keypem "here")), make sure keys and tpcds-spark can be accessed on each K8S node 5. Start a bigdl-ppml enabled Spark K8S client container with configured local IP, key, tpc-ds and kuberconfig path -```bash -export ENCLAVE_KEY=/YOUR_DIR/keys/enclave-key.pem -export DATA_PATH=/YOUR_DIR/zoo-tutorials/tpcds-spark -export KEYS_PATH=/YOUR_DIR/keys -export SECURE_PASSWORD_PATH=/YOUR_DIR/password -export KUBERCONFIG_PATH=/YOUR_DIR/kuberconfig -export LOCAL_IP=$local_ip -export DOCKER_IMAGE=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT -sudo docker run -itd \ - --privileged \ - --net=host \ - --name=spark-local-k8s-client \ - --oom-kill-disable \ - --device=/dev/sgx/enclave \ - --device=/dev/sgx/provision \ - -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ - -v $ENCLAVE_KEY:/graphene/Pal/src/host/Linux-SGX/signer/enclave-key.pem \ - -v $DATA_PATH:/ppml/trusted-big-data-ml/work/tpcds-spark \ - -v $KEYS_PATH:/ppml/trusted-big-data-ml/work/keys \ - -v $SECURE_PASSWORD_PATH:/ppml/trusted-big-data-ml/work/password \ - -v $KUBERCONFIG_PATH:/root/.kube/config \ - -e RUNTIME_SPARK_MASTER=k8s://https://$LOCAL_IP:6443 \ - -e RUNTIME_K8S_SERVICE_ACCOUNT=spark \ - -e RUNTIME_K8S_SPARK_IMAGE=$DOCKER_IMAGE \ - -e RUNTIME_DRIVER_HOST=$LOCAL_IP \ - -e RUNTIME_DRIVER_PORT=54321 \ - -e RUNTIME_EXECUTOR_INSTANCES=1 \ - -e RUNTIME_EXECUTOR_CORES=4 \ - -e RUNTIME_EXECUTOR_MEMORY=20g \ - -e RUNTIME_TOTAL_EXECUTOR_CORES=4 \ - -e RUNTIME_DRIVER_CORES=4 \ - -e RUNTIME_DRIVER_MEMORY=10g \ - -e SGX_MEM_SIZE=64G \ - -e SGX_LOG_LEVEL=error \ - -e LOCAL_IP=$LOCAL_IP \ - $DOCKER_IMAGE bash -``` + ```bash + export ENCLAVE_KEY=/YOUR_DIR/keys/enclave-key.pem + export DATA_PATH=/YOUR_DIR/zoo-tutorials/tpcds-spark + export KEYS_PATH=/YOUR_DIR/keys + export SECURE_PASSWORD_PATH=/YOUR_DIR/password + export KUBERCONFIG_PATH=/YOUR_DIR/kuberconfig + export LOCAL_IP=$local_ip + export DOCKER_IMAGE=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT + sudo docker run -itd \ + --privileged \ + --net=host \ + --name=spark-local-k8s-client \ + --oom-kill-disable \ + --device=/dev/sgx/enclave \ + --device=/dev/sgx/provision \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + -v $ENCLAVE_KEY:/graphene/Pal/src/host/Linux-SGX/signer/enclave-key.pem \ + -v $DATA_PATH:/ppml/trusted-big-data-ml/work/tpcds-spark \ + -v $KEYS_PATH:/ppml/trusted-big-data-ml/work/keys \ + -v $SECURE_PASSWORD_PATH:/ppml/trusted-big-data-ml/work/password \ + -v $KUBERCONFIG_PATH:/root/.kube/config \ + -e RUNTIME_SPARK_MASTER=k8s://https://$LOCAL_IP:6443 \ + -e RUNTIME_K8S_SERVICE_ACCOUNT=spark \ + -e RUNTIME_K8S_SPARK_IMAGE=$DOCKER_IMAGE \ + -e RUNTIME_DRIVER_HOST=$LOCAL_IP \ + -e RUNTIME_DRIVER_PORT=54321 \ + -e RUNTIME_EXECUTOR_INSTANCES=1 \ + -e RUNTIME_EXECUTOR_CORES=4 \ + -e RUNTIME_EXECUTOR_MEMORY=20g \ + -e RUNTIME_TOTAL_EXECUTOR_CORES=4 \ + -e RUNTIME_DRIVER_CORES=4 \ + -e RUNTIME_DRIVER_MEMORY=10g \ + -e SGX_MEM_SIZE=64G \ + -e SGX_LOG_LEVEL=error \ + -e LOCAL_IP=$LOCAL_IP \ + $DOCKER_IMAGE bash + ``` 6. Attach to the client container -```bash -sudo docker exec -it spark-local-k8s-client bash -``` + ```bash + sudo docker exec -it spark-local-k8s-client bash + ``` 7. Modify `spark-executor-template.yaml`, add path of `enclave-key`, `tpcds-spark` and `kuberconfig` on host -```yaml -apiVersion: v1 -kind: Pod -spec: - containers: - - name: spark-executor - securityContext: - privileged: true - volumeMounts: - ... - - name: tpcds - mountPath: /ppml/trusted-big-data-ml/work/tpcds-spark - - name: kubeconf - mountPath: /root/.kube/config - volumes: - - name: enclave-key - hostPath: - path: /root/keys/enclave-key.pem - ... - - name: tpcds - hostPath: - path: /path/to/tpcds-spark - - name: kubeconf - hostPath: - path: /path/to/kuberconfig -``` + ```yaml + apiVersion: v1 + kind: Pod + spec: + containers: + - name: spark-executor + securityContext: + privileged: true + volumeMounts: + ... + - name: tpcds + mountPath: /ppml/trusted-big-data-ml/work/tpcds-spark + - name: kubeconf + mountPath: /root/.kube/config + volumes: + - name: enclave-key + hostPath: + path: /root/keys/enclave-key.pem + ... + - name: tpcds + hostPath: + path: /path/to/tpcds-spark + - name: kubeconf + hostPath: + path: /path/to/kuberconfig + ``` 8. Execute TPC-DS queries -Optional argument `QUERY` is the query number to run. Multiple query numbers should be separated by space, e.g. `1 2 3`. If no query number is specified, all 1-99 queries would be executed. + Optional argument `QUERY` is the query number to run. Multiple query numbers should be separated by space, e.g. `1 2 3`. If no query number is specified, all 1-99 queries would be executed. -```bash -secure_password=`openssl rsautl -inkey /ppml/trusted-big-data-ml/work/password/key.txt -decrypt /performance` directory. + After benchmark is finished, the performance result is saved as `part-*.csv` file under `/performance` directory. diff --git a/docs/readthedocs/source/doc/PPML/QuickStart/tpc-h_with_sparksql_on_k8s.md b/docs/readthedocs/source/doc/PPML/QuickStart/tpc-h_with_sparksql_on_k8s.md index 141cfce6..5b89ed9b 100644 --- a/docs/readthedocs/source/doc/PPML/QuickStart/tpc-h_with_sparksql_on_k8s.md +++ b/docs/readthedocs/source/doc/PPML/QuickStart/tpc-h_with_sparksql_on_k8s.md @@ -8,198 +8,198 @@ ### Prepare TPC-H kit and data ### 1. Generate data - Go to [TPC Download](https://www.tpc.org/tpc_documents_current_versions/current_specifications5.asp) site, choose `TPC-H` source code, then download the TPC-H toolkits. **Follow the download instructions carefully.** - After you download the tpc-h tools zip and uncompressed the zip file. Go to `dbgen` directory, and create `makefile` based on `makefile.suite`, and modify `makefile` according to the prompts inside, and run `make`. + Go to [TPC Download](https://www.tpc.org/tpc_documents_current_versions/current_specifications5.asp) site, choose `TPC-H` source code, then download the TPC-H toolkits. **Follow the download instructions carefully.** + After you download the tpc-h tools zip and uncompressed the zip file. Go to `dbgen` directory, and create `makefile` based on `makefile.suite`, and modify `makefile` according to the prompts inside, and run `make`. - This should generate an executable called `dbgen` - ``` - ./dbgen -h - ``` + This should generate an executable called `dbgen` + ``` + ./dbgen -h + ``` - gives you the various options for generating the tables. The simplest case is running: - ``` - ./dbgen - ``` - which generates tables with extension `.tbl` with scale 1 (default) for a total of rougly 1GB size across all tables. For different size tables you can use the `-s` option: - ``` - ./dbgen -s 10 - ``` - will generate roughly 10GB of input data. + gives you the various options for generating the tables. The simplest case is running: + ``` + ./dbgen + ``` + which generates tables with extension `.tbl` with scale 1 (default) for a total of rougly 1GB size across all tables. For different size tables you can use the `-s` option: + ``` + ./dbgen -s 10 + ``` + will generate roughly 10GB of input data. - You need to move all .tbl files to a new directory as raw data. + You need to move all .tbl files to a new directory as raw data. - You can then either upload your data to remote file system or read them locally. + You can then either upload your data to remote file system or read them locally. 2. Encrypt Data - Encrypt data with specified Key Management Service (`SimpleKeyManagementService`, or `EHSMKeyManagementService` , or `AzureKeyManagementService`). Details can be found here: https://github.com/intel-analytics/BigDL/tree/main/ppml/services/kms-utils/docker + Encrypt data with specified Key Management Service (`SimpleKeyManagementService`, or `EHSMKeyManagementService` , or `AzureKeyManagementService`). Details can be found here: https://github.com/intel-analytics/BigDL/tree/main/ppml/services/kms-utils/docker - The example code of encrypt data with `SimpleKeyManagementService` is like below: - ``` - java -cp "$BIGDL_HOME/jars/bigdl-ppml-spark_3.1.2-2.1.0-SNAPSHOT.jar:$SPARK_HOME/conf/:$SPARK_HOME/jars/*:$BIGDL_HOME/jars/*" \ - -Xmx10g \ - com.intel.analytics.bigdl.ppml.examples.tpch.EncryptFiles \ - --inputPath xxx/dbgen-input \ - --outputPath xxx/dbgen-encrypted - --kmsType SimpleKeyManagementService - --simpleAPPID xxxxxxxxxxxx \ - --simpleAPPKEY xxxxxxxxxxxx \ - --primaryKeyPath /path/to/simple_encrypted_primary_key \ - --dataKeyPath /path/to/simple_encrypted_data_key - ``` + The example code of encrypt data with `SimpleKeyManagementService` is like below: + ``` + java -cp "$BIGDL_HOME/jars/bigdl-ppml-spark_3.1.2-2.1.0-SNAPSHOT.jar:$SPARK_HOME/conf/:$SPARK_HOME/jars/*:$BIGDL_HOME/jars/*" \ + -Xmx10g \ + com.intel.analytics.bigdl.ppml.examples.tpch.EncryptFiles \ + --inputPath xxx/dbgen-input \ + --outputPath xxx/dbgen-encrypted + --kmsType SimpleKeyManagementService + --simpleAPPID xxxxxxxxxxxx \ + --simpleAPPKEY xxxxxxxxxxxx \ + --primaryKeyPath /path/to/simple_encrypted_primary_key \ + --dataKeyPath /path/to/simple_encrypted_data_key + ``` ### Deploy PPML TPC-H on Kubernetes ### -1. Pull docker image - ``` - sudo docker pull intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT - ``` +1. Pull docker image + ``` + sudo docker pull intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT + ``` 2. Prepare SGX keys (following instructions [here](https://github.com/intel-analytics/BigDL/tree/main/ppml/trusted-big-data-ml/python/docker-graphene#11-prepare-the-keyspassworddataenclave-keypem "here")), make sure keys and tpch-spark can be accessed on each K8S node 3. Start a bigdl-ppml enabled Spark K8S client container with configured local IP, key, tpch and kuberconfig path - ``` - export ENCLAVE_KEY=/path/to/enclave-key.pem - export SECURE_PASSWORD_PATH=/path/to/password - export DATA_PATH=/path/to/data - export KEYS_PATH=/path/to/keys - export KUBERCONFIG_PATH=/path/to/kuberconfig - export LOCAL_IP=$local_ip - export DOCKER_IMAGE=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT - sudo docker run -itd \ - --privileged \ - --net=host \ - --name=spark-local-k8s-client \ - --oom-kill-disable \ - --device=/dev/sgx/enclave \ - --device=/dev/sgx/provision \ - -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ - -v $SECURE_PASSWORD_PATH:/ppml/trusted-big-data-ml/work/password \ - -v $ENCLAVE_KEY:/graphene/Pal/src/host/Linux-SGX/signer/enclave-key.pem \ - -v $DATA_PATH:/ppml/trusted-big-data-ml/work/data \ - -v $KEYS_PATH:/ppml/trusted-big-data-ml/work/keys \ - -v $KUBERCONFIG_PATH:/root/.kube/config \ - -e RUNTIME_SPARK_MASTER=k8s://https://$LOCAL_IP:6443 \ - -e RUNTIME_K8S_SERVICE_ACCOUNT=spark \ - -e RUNTIME_K8S_SPARK_IMAGE=$DOCKER_IMAGE \ - -e RUNTIME_DRIVER_HOST=$LOCAL_IP \ - -e RUNTIME_DRIVER_PORT=54321 \ - -e RUNTIME_EXECUTOR_INSTANCES=1 \ - -e RUNTIME_EXECUTOR_CORES=4 \ - -e RUNTIME_EXECUTOR_MEMORY=20g \ - -e RUNTIME_TOTAL_EXECUTOR_CORES=4 \ - -e RUNTIME_DRIVER_CORES=4 \ - -e RUNTIME_DRIVER_MEMORY=10g \ - -e SGX_MEM_SIZE=64G \ - -e SGX_LOG_LEVEL=error \ - -e LOCAL_IP=$LOCAL_IP \ - $DOCKER_IMAGE bash - ``` + ``` + export ENCLAVE_KEY=/path/to/enclave-key.pem + export SECURE_PASSWORD_PATH=/path/to/password + export DATA_PATH=/path/to/data + export KEYS_PATH=/path/to/keys + export KUBERCONFIG_PATH=/path/to/kuberconfig + export LOCAL_IP=$local_ip + export DOCKER_IMAGE=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene:2.1.0-SNAPSHOT + sudo docker run -itd \ + --privileged \ + --net=host \ + --name=spark-local-k8s-client \ + --oom-kill-disable \ + --device=/dev/sgx/enclave \ + --device=/dev/sgx/provision \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + -v $SECURE_PASSWORD_PATH:/ppml/trusted-big-data-ml/work/password \ + -v $ENCLAVE_KEY:/graphene/Pal/src/host/Linux-SGX/signer/enclave-key.pem \ + -v $DATA_PATH:/ppml/trusted-big-data-ml/work/data \ + -v $KEYS_PATH:/ppml/trusted-big-data-ml/work/keys \ + -v $KUBERCONFIG_PATH:/root/.kube/config \ + -e RUNTIME_SPARK_MASTER=k8s://https://$LOCAL_IP:6443 \ + -e RUNTIME_K8S_SERVICE_ACCOUNT=spark \ + -e RUNTIME_K8S_SPARK_IMAGE=$DOCKER_IMAGE \ + -e RUNTIME_DRIVER_HOST=$LOCAL_IP \ + -e RUNTIME_DRIVER_PORT=54321 \ + -e RUNTIME_EXECUTOR_INSTANCES=1 \ + -e RUNTIME_EXECUTOR_CORES=4 \ + -e RUNTIME_EXECUTOR_MEMORY=20g \ + -e RUNTIME_TOTAL_EXECUTOR_CORES=4 \ + -e RUNTIME_DRIVER_CORES=4 \ + -e RUNTIME_DRIVER_MEMORY=10g \ + -e SGX_MEM_SIZE=64G \ + -e SGX_LOG_LEVEL=error \ + -e LOCAL_IP=$LOCAL_IP \ + $DOCKER_IMAGE bash + ``` 4. Attach to the client container - ``` - sudo docker exec -it spark-local-k8s-client bash - ``` + ``` + sudo docker exec -it spark-local-k8s-client bash + ``` 5. Modify `spark-executor-template.yaml`, add path of `enclave-key`, `tpch-spark` and `kuberconfig` on host - ``` - apiVersion: v1 - kind: Pod - spec: - containers: - - name: spark-executor - securityContext: - privileged: true - volumeMounts: - ... - - name: tpch - mountPath: /ppml/trusted-big-data-ml/work/tpch-spark - - name: kubeconf - mountPath: /root/.kube/config - volumes: - - name: enclave-key - hostPath: - path: /root/keys/enclave-key.pem - ... - - name: tpch - hostPath: - path: /path/to/tpch-spark - - name: kubeconf - hostPath: - path: /path/to/kuberconfig - ``` + ``` + apiVersion: v1 + kind: Pod + spec: + containers: + - name: spark-executor + securityContext: + privileged: true + volumeMounts: + ... + - name: tpch + mountPath: /ppml/trusted-big-data-ml/work/tpch-spark + - name: kubeconf + mountPath: /root/.kube/config + volumes: + - name: enclave-key + hostPath: + path: /root/keys/enclave-key.pem + ... + - name: tpch + hostPath: + path: /path/to/tpch-spark + - name: kubeconf + hostPath: + path: /path/to/kuberconfig + ``` 6. Run PPML TPC-H - ```bash - secure_password=`openssl rsautl -inkey /ppml/trusted-big-data-ml/work/password/key.txt -decrypt Q01 39.80204010 \ No newline at end of file + The result is in OUTPUT_DIR. There should be a file called TIMES.TXT with content formatted like: + >Q01 39.80204010 \ No newline at end of file diff --git a/docs/readthedocs/source/doc/PPML/index.rst b/docs/readthedocs/source/doc/PPML/index.rst new file mode 100644 index 00000000..b60a5403 --- /dev/null +++ b/docs/readthedocs/source/doc/PPML/index.rst @@ -0,0 +1,71 @@ +BigDL-PPML +========================= + +Protecting privacy and confidentiality is critical for large-scale data analysis and machine learning. BigDL PPML (BigDL Privacy Preserving Machine Learning) combines various low-level hardware and software security technologies (e.g., Intel® Software Guard Extensions (Intel® SGX), Security Key Management, Remote Attestation, Data Encryption, Federated Learning, etc.) so that users can continue applying standard Big Data and AI technologies (such as Apache Spark, Apache Flink, TensorFlow, PyTorch, etc.) without sacrificing privacy. + +---------------------- + + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + Documents in these sections helps you getting started quickly with PPML. + + +++ + + :bdg-link:`Introduction <./Overview/intro.html>` | + :bdg-link:`Hello World Example <./Overview/quicktour.html>` + + + .. grid-item-card:: + + **User Guide** + ^^^ + + Provides you with in-depth information about PPML features and concepts and step-by-step guides. + + +++ + + :bdg-link:`User Guide <./Overview/userguide.html>` | + :bdg-link:`Advanced Topics <./Overview/misc.html>` + + + .. grid-item-card:: + + **Tutorials** + ^^^ + + PPML Tutorials and Examples. + + +++ + + :bdg-link:`End-to-End Example <./Overview/examples.html>` | + :bdg-link:`More Examples ` + + .. grid-item-card:: + + **Videos** + ^^^ + + Videos and Demos helps you quick understand the architecture and start hands-on work. + + +++ + + :bdg-link:`Introduction <./Overview/intro.html#what-is-bigdl-ppml>` | + :bdg-link:`E2E Workflow <./QuickStart/end-to-end.html#e2e-architecture-overview>` | + :bdg-link:`E2E Demo <./QuickStart/end-to-end.html#video-demo>` + + + + + + +.. toctree:: + :hidden: + + BigDL-PPML Document \ No newline at end of file diff --git a/docs/readthedocs/source/doc/DLlib/Overview/clipping.md b/docs/readthedocs/source/doc/PythonAPI/DLlib/clipping.md similarity index 95% rename from docs/readthedocs/source/doc/DLlib/Overview/clipping.md rename to docs/readthedocs/source/doc/PythonAPI/DLlib/clipping.md index 16def661..038667db 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/clipping.md +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/clipping.md @@ -1,3 +1,7 @@ +# Clipping + +-------- + ## ConstantGradientClipping ## Set constant gradient clipping during the training process. diff --git a/docs/readthedocs/source/doc/PythonAPI/DLlib/core_layers.md b/docs/readthedocs/source/doc/PythonAPI/DLlib/core_layers.md new file mode 100644 index 00000000..9c83648a --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/core_layers.md @@ -0,0 +1,3328 @@ +# Core Layers API + +-------- + +This section describes all the available layers in the Keras-like API. + +To set the name of a specific layer, you call the method `setName(name)` of that layer. + +------------------ + +### Masking + +Use a mask value to skip timesteps for a sequence. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Masking(maskValue = 0.0, inputShape = null) + + + .. tab:: Python API + + .. code-block:: python + + Masking(mask_value=0.0, input_shape=None, name=None) + +``` + +**Parameters:** + +* `maskValue`: Mask value. For each timestep in the input (the second dimension), if all the values in the input at that timestep are equal to 'maskValue', then the timestep will be masked (skipped) in all downstream layers. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Masking + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Masking[Float](inputShape = Shape(3))) + val input = Tensor[Float](2, 3).randn() + val output = model.forward(input) + + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + 1.4539868 1.5623108 -1.4101523 + 0.77073747 -0.18994702 2.2574463 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + 1.4539868 1.5623108 -1.4101523 + 0.77073747 -0.18994702 2.2574463 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.nn.keras.topology import Sequential + from bigdl.nn.keras.layer import Masking + + model = Sequential() + model.add(Masking(input_shape=(3, ))) + input = np.random.random([2, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[0.31542103 0.20640659 0.22282763] + [0.99352167 0.90135718 0.24504717]] + + Output is + + .. code-block:: python + + [[0.31542102 0.2064066 0.22282763] + [0.9935217 0.9013572 0.24504717]] + +``` + +--- + +### SparseDense + + +SparseDense is the sparse version of layer Dense. SparseDense has two different from Dense: +firstly, SparseDense's input Tensor is a SparseTensor. Secondly, SparseDense doesn't backward +gradient to next layer in the backpropagation by default, as the gradInput of SparseDense is +useless and very big in most cases. + +But, considering model like Wide&Deep, we provide backwardStart and backwardLength to backward +part of the gradient to next layer. + +The most common input is 2D. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + SparseDense(outputDim, init = "glorot_uniform", activation = null, wRegularizer = null, bRegularizer = null, backwardStart = -1, backwardLength = -1, initWeight = null, initBias = null, initGradWeight = null, initGradBias = null, bias = true, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + SparseDense(output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, backward_start=-1, backward_length=-1, init_weight=None, init_bias=None, init_grad_weight=None, init_grad_bias=None, bias=True, input_shape=None, name=None) + +``` + +**Parameters:** + +* `outputDim`: The size of the output dimension. +* `init`: String representation of the initialization method for the weights of the layer. Default is 'glorot_uniform'. +* `activation`: String representation of the activation function to use. Default is null. +* `wRegularizer`: An instance of [Regularizer], applied to the input weights matrices. Default is null. +* `bRegularizer`: An instance of [Regularizer], applied to the bias. Default is null. +* `bias`: Whether to include a bias (i.e. make the layer affine rather than linear). Default is true. +* `backwardStart`: Backward start index, counting from 1. +* `backwardLength`: Backward length. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a `Shape` object. For Python API, it should be a shape tuple. Batch dimension should be excluded. +* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. + + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.SparseDense + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val layer = SparseDense[Float](outputDim = 5, inputShape = Shape(2, 4)) + layer.build(Shape(-1, 2, 4)) + val input = Tensor[Float](Array(2, 4)).rand() + input.setValue(1, 1, 1f) + input.setValue(2, 3, 3f) + val sparseInput = Tensor.sparse(input) + val output = layer.forward(sparseInput) + + + Input is: + + .. code-block:: scala + + input: + (0, 0) : 1.0 + (0, 1) : 0.2992794 + (0, 2) : 0.11227019 + (0, 3) : 0.722947 + (1, 0) : 0.6147614 + (1, 1) : 0.4288646 + (1, 2) : 3.0 + (1, 3) : 0.7749917 + [com.intel.analytics.bigdl.tensor.SparseTensor of size 2x4] + + Output is: + + .. code-block:: scala + + output: + 0.053516 0.33429605 0.22587383 -0.8998945 0.24308181 + 0.76745665 -1.614114 0.5381658 -2.2226436 -0.15573677 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x5] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + from bigdl.dllib.utils.common import JTensor + + model = Sequential() + model.add(SparseDense(output_dim=2, input_shape=(3, 4))) + input = JTensor.sparse( + a_ndarray=np.array([1, 3, 2, 4]), + i_ndarray = np.array([[0, 0, 1, 2], + [0, 3, 2, 1]]), + shape = np.array([3, 4]) + ) + output = model.forward(input) + + Input is: + + .. code-block:: python + + JTensor: storage: [1. 3. 2. 4.], shape: [3 4] ,indices [[0 0 1 2] + [0 3 2 1]], float + + Output is + + .. code-block:: python + + [[ 1.57136 2.29596 ] + [ 0.5791738 -1.6598101 ] + [ 2.331141 -0.84687066]] + +``` + +--- + +### SoftShrink + + +Applies the soft shrinkage function element-wise to the input. + +When you use this layer as the first layer of a model, you need to provide +the argument inputShape (a Single Shape, does not include the batch dimension). + +Remark: This layer is from Torch and wrapped in Keras style. + + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + SoftShrink(value = 0.5, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + SoftShrink(value = 0.5, input_shape=None, name=None) + +``` + +**Parameters:** + +* `value`: value The threshold value. Default is 0.5. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a `Shape` object. For Python API, it should be a shape tuple. Batch dimension should be excluded. +* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.SoftShrink + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(SoftShrink[Float](0.6, inputShape = Shape(2, 3, 4))) + val input = Tensor[Float](2, 2, 3, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + -0.36938807 0.023556225 -1.1655436 -0.34449077 + 0.9444338 -0.086538695 -1.0425501 1.364976 + -1.2563878 -0.1842559 0.43428117 1.0756494 + + (1,2,.,.) = + -0.19888283 1.251872 0.114836805 -0.6208773 + 0.0051822234 -0.8998633 0.06937465 -0.3929931 + -0.1058129 0.6945743 -0.40083578 -0.6252444 + + (2,1,.,.) = + -0.9899709 -0.77926594 -0.15497442 -0.15031165 + -0.6028622 0.86623466 -2.1543107 0.41970536 + -0.8215522 0.3014275 -0.32184362 0.14445356 + + (2,2,.,.) = + 0.74701905 0.10044397 -0.40519297 0.03822808 + 0.30726334 0.27862388 1.731753 0.032177072 + -1.3476961 -0.2294767 0.99794704 0.7398458 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.0 0.0 -0.56554353 0.0 + 0.34443378 0.0 -0.44255006 0.764976 + -0.6563878 0.0 0.0 0.47564936 + + (1,2,.,.) = + 0.0 0.6518719 0.0 -0.020877302 + 0.0 -0.29986328 0.0 0.0 + 0.0 0.09457427 0.0 -0.025244355 + + (2,1,.,.) = + -0.3899709 -0.17926592 0.0 0.0 + -0.0028621554 0.26623464 -1.5543107 0.0 + -0.2215522 0.0 0.0 0.0 + + (2,2,.,.) = + 0.14701903 0.0 0.0 0.0 + 0.0 0.0 1.131753 0.0 + -0.74769604 0.0 0.397947 0.13984579 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(SoftShrink(0.6, input_shape=(2, 3, 4))) + input = np.random.random([2, 2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[[ 0.43421006, 0.28394451, 0.15221226, 0.47268966], + [ 0.22426224, 0.24855662, 0.790498 , 0.67767582], + [ 0.14879562, 0.56077882, 0.61470262, 0.94875862]], + + [[ 0.72404932, 0.89780875, 0.08456734, 0.01303937], + [ 0.25023568, 0.45392504, 0.587254 , 0.51164461], + [ 0.12277567, 0.05571182, 0.17076456, 0.71660884]]], + + + [[[ 0.06369975, 0.85395557, 0.35752425, 0.606633 ], + [ 0.67640252, 0.86861737, 0.18040722, 0.55467108], + [ 0.24102058, 0.37580645, 0.81601612, 0.56513788]], + + [[ 0.8461435 , 0.65668365, 0.17969807, 0.51602926], + [ 0.86191073, 0.34245714, 0.62795207, 0.36706125], + [ 0.80344028, 0.81056003, 0.80959083, 0.15366483]]]]) + + Output is + + .. code-block:: python + + array([[[[ 0. , 0. , 0. , 0. ], + [ 0. , 0. , 0.19049799, 0.07767582], + [ 0. , 0. , 0.01470262, 0.34875858]], + + [[ 0.12404931, 0.29780871, 0. , 0. ], + [ 0. , 0. , 0. , 0. ], + [ 0. , 0. , 0. , 0.1166088 ]]], + + + [[[ 0. , 0.25395554, 0. , 0.00663298], + [ 0.07640249, 0.26861733, 0. , 0. ], + [ 0. , 0. , 0.21601611, 0. ]], + + [[ 0.24614346, 0.05668366, 0. , 0. ], + [ 0.26191074, 0. , 0.02795208, 0. ], + [ 0.20344025, 0.21056002, 0.20959079, 0. ]]]], dtype=float32) + + ``` + +--- +### Reshape +Reshapes an output to a certain shape. + +Supports shape inference by allowing one -1 in the target shape. For example, if input shape is (2, 3, 4), target shape is (3, -1), then output shape will be (3, 8). + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Reshape(targetShape, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Reshape(target_shape, input_shape=None, name=None) +``` + +**Parameters:** + +* `targetShape`: The target shape that you desire to have. Batch dimension should be excluded. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Reshape + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Reshape(Array(3, 8), inputShape = Shape(2, 3, 4))) + val input = Tensor[Float](2, 2, 3, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + -1.7092276 -1.3941092 -0.6348466 0.71309644 + 0.3605411 0.025597548 0.4287048 -0.548675 + 0.4623341 -2.3912702 0.22030865 -0.058272455 + + (1,2,.,.) = + -1.5049093 -1.8828062 0.8230564 -0.020209199 + -0.3415721 1.1219939 1.1089007 -0.74697906 + -1.503861 -1.616539 0.048006497 1.1613717 + + (2,1,.,.) = + 0.21216023 1.0107462 0.8586909 -0.05644316 + -0.31436008 1.6892323 -0.9961186 -0.08169463 + 0.3559391 0.010261055 -0.70408463 -1.2480727 + + (2,2,.,.) = + 1.7663039 0.07122444 0.073556066 -0.7847014 + 0.17604464 -0.99110585 -1.0302067 -0.39024687 + -0.0260166 -0.43142694 0.28443158 0.72679126 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + -1.7092276 -1.3941092 -0.6348466 0.71309644 0.3605411 0.025597548 0.4287048 -0.548675 + 0.4623341 -2.3912702 0.22030865 -0.058272455 -1.5049093 -1.8828062 0.8230564 -0.020209199 + -0.3415721 1.1219939 1.1089007 -0.74697906 -1.503861 -1.616539 0.048006497 1.1613717 + + (2,.,.) = + 0.21216023 1.0107462 0.8586909 -0.05644316 -0.31436008 1.6892323 -0.9961186 -0.08169463 + 0.3559391 0.010261055 -0.70408463 -1.2480727 1.7663039 0.07122444 0.073556066 -0.7847014 + 0.17604464 -0.99110585 -1.0302067 -0.39024687 -0.0260166 -0.43142694 0.28443158 0.72679126 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x8] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + + model = Sequential() + model.add(Reshape(target_shape=(3, 8), input_shape=(2, 3, 4))) + input = np.random.random([2, 2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[[0.39260304 0.10383185 0.87490319 0.89167328] + [0.61649117 0.43285247 0.86851582 0.97743004] + [0.90018969 0.04303951 0.74263493 0.14208656]] + [[0.66193405 0.93432157 0.76160537 0.70437459] + [0.99953431 0.23016734 0.42293405 0.66078049] + [0.03357645 0.9695145 0.30111138 0.67109948]]] + + [[[0.39640201 0.92930203 0.86027666 0.13958544] + [0.34584767 0.14743425 0.93804016 0.38053062] + [0.55068792 0.77375329 0.84161166 0.48131356]] + [[0.90116368 0.53253689 0.03332962 0.58278686] + [0.34935685 0.32599554 0.97641892 0.57696434] + [0.53974677 0.90682861 0.20027319 0.05962118]]]] + ``` + + Output is: + + .. code-block:: python + + [[[0.39260304 0.10383185 0.8749032 0.89167327 0.6164912 0.43285248 0.86851585 0.97743005] + [0.9001897 0.04303951 0.74263495 0.14208655 0.661934 0.9343216 0.7616054 0.7043746 ] + [0.9995343 0.23016734 0.42293406 0.6607805 0.03357645 0.9695145 0.30111137 0.6710995 ]] + + [[0.396402 0.92930204 0.86027664 0.13958544 0.34584767 0.14743425 0.93804014 0.38053063] + [0.5506879 0.7737533 0.8416117 0.48131356 0.9011637 0.53253686 0.03332962 0.58278686] + [0.34935686 0.32599553 0.9764189 0.5769643 0.53974676 0.9068286 0.20027319 0.05962119]]] +``` + +--- +### Merge + +Used to merge a list of inputs into a single output, following some merge mode. + +Merge must have at least two input layers. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Merge(layers = null, mode = "sum", concatAxis = -1, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Merge(layers=None, mode="sum", concat_axis=-1, input_shape=None, name=None) + +``` + +**Parameters:** + +* `layers`: A list of layer instances. Must be more than one layer. +* `mode`: Merge mode. String, must be one of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'. Default is 'sum'. +* `concatAxis`: Integer, axis to use when concatenating layers. Only specify this when merge mode is 'concat'. Default is -1, meaning the last axis of the input. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object. For Python API, it should be a list of shape tuple. Batch dimension should be excluded. + + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.InputLayer + import com.intel.analytics.bigdl.dllib.keras.layers.Merge + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.utils.{Shape, T} + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + val l1 = InputLayer[Float](inputShape = Shape(2, 3)) + val l2 = InputLayer[Float](inputShape = Shape(2, 3)) + val layer = Merge[Float](layers = List(l1, l2), mode = "sum") + model.add(layer) + val input1 = Tensor[Float](2, 2, 3).rand(0, 1) + val input2 = Tensor[Float](2, 2, 3).rand(0, 1) + val input = T(1 -> input1, 2 -> input2) + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.utils.Table = + { + 2: (1,.,.) = + 0.87815475 0.15025006 0.34412447 + 0.07909282 0.008027249 0.111715704 + + (2,.,.) = + 0.52245367 0.2547527 0.35857987 + 0.7718501 0.26783863 0.8642062 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + 1: (1,.,.) = + 0.5377018 0.28364193 0.3424284 + 0.0075349305 0.9018168 0.9435114 + + (2,.,.) = + 0.09112563 0.88585275 0.3100201 + 0.7910178 0.57497376 0.39764535 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + } + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 1.4158566 0.433892 0.6865529 + 0.08662775 0.90984404 1.0552272 + + (2,.,.) = + 0.6135793 1.1406054 0.66859996 + 1.5628679 0.8428124 1.2618515 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + l1 = InputLayer(input_shape=(3, 4)) + l2 = InputLayer(input_shape=(3, 4)) + model.add(Merge(layers=[l1, l2], mode='sum')) + input = [np.random.random([2, 3, 4]), np.random.random([2, 3, 4])] + output = model.forward(input) + + + Input is: + + .. code-block:: python + + [[[[0.28764351, 0.0236015 , 0.78927442, 0.52646492], + [0.63922826, 0.45101604, 0.4555552 , 0.70105653], + [0.75790798, 0.78551523, 0.00686686, 0.61290369]], + + [[0.00430865, 0.3303661 , 0.59915782, 0.90362298], + [0.26230717, 0.99383052, 0.50630521, 0.99119486], + [0.56138318, 0.68165639, 0.10644523, 0.51860127]]], + + [[[0.84365767, 0.8854741 , 0.84183673, 0.96322321], + [0.49354248, 0.97936826, 0.2266097 , 0.88083622], + [0.11011776, 0.65762034, 0.17446099, 0.76658969]], + + [[0.58266689, 0.86322199, 0.87122999, 0.19031255], + [0.42275118, 0.76379413, 0.21355413, 0.81132937], + [0.97294728, 0.68601731, 0.39871792, 0.63172344]]]] + + Output is + + .. code-block:: python + + [[[1.1313012 0.90907556 1.6311111 1.4896882 ] + [1.1327708 1.4303843 0.6821649 1.5818927 ] + [0.8680257 1.4431355 0.18132785 1.3794935 ]] + + [[0.5869755 1.1935881 1.4703878 1.0939355 ] + [0.68505836 1.7576246 0.71985936 1.8025242 ] + [1.5343305 1.3676738 0.50516313 1.1503248 ]]] + +``` + +--- +### MaxoutDense + +A dense maxout layer that takes the element-wise maximum of linear layers. + +This allows the layer to learn a convex, piecewise linear activation function over the inputs. + +The input of this layer should be 2D. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + MaxoutDense(outputDim, nbFeature = 4, wRegularizer = null, bRegularizer = null, bias = true, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + MaxoutDense(output_dim, nb_feature=4, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, name=None) +``` + + +**Parameters:** + +* `outputDim`: The size of output dimension. +* `nbFeature`: Number of Dense layers to use internally. Integer. Default is 4. +* `wRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), (eg. L1 or L2 regularization), applied to the input weights matrices. Default is null. +* `bRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the bias. Default is null. +* `bias`: Whether to include a bias (i.e. make the layer affine rather than linear). Default is true. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.MaxoutDense + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(MaxoutDense(2, inputShape = Shape(3))) + val input = Tensor[Float](2, 3).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + -1.3550005 -1.1668127 -1.2882779 + 0.83600295 -1.94683 1.323666 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + Output is: + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + 0.71675766 1.2987505 + 0.9871184 0.6634239 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(MaxoutDense(2, input_shape=(3, ))) + input = np.random.random([2, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[0.15996114 0.8391686 0.81922903] + [0.52929427 0.35061754 0.88167693]] + + Output is + + .. code-block:: python + + [[0.4479192 0.4842512] + [0.16833156 0.521764 ]] +``` + +--- +### Squeeze +Delete the singleton dimension(s). The batch dimension needs to be unchanged. + +For example, if input has size (2, 1, 3, 4, 1): + +Squeeze(1) will give output size (2, 3, 4, 1), + +Squeeze() will give output size (2, 3, 4) + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Squeeze(dims = null, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Squeeze(dim=None, input_shape=None, name=None) +``` + +**Parameters:** + +* `dims`: The dimension(s) to squeeze. 0-based index. Cannot squeeze the batch dimension. The selected dimensions must be singleton, i.e. having size 1. Default is null, and in this case all the non-batch singleton dimensions will be deleted. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Squeeze + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Squeeze[Float](1, inputShape = Shape(1, 1, 32))) + val input = Tensor[Float](1, 1, 1, 32).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + 0.5521966 -1.2199087 0.365958 1.3845297 0.115254946 -0.20352958 2.4912808 0.987046 -0.2115477 3.0530396 -1.0043625 1.4688021 -1.2412603 -0.25383064 0.49164283 -0.40329486 0.26323202 0.7979045 0.025444122 0.47221214 1.3995043 0.48498031 -0.86961967 -0.058370713 -0.85965866 -1.2727696 0.45570874 0.73393697 0.2567143 1.4261572 -0.37773672 -0.7339463 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 1x1x1x32] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 0.5521966 -1.2199087 0.365958 1.3845297 0.115254946 -0.20352958 2.4912808 0.987046 -0.2115477 3.0530396 -1.0043625 1.4688021 -1.2412603 -0.25383064 0.49164283 -0.40329486 0.26323202 0.7979045 0.025444122 0.47221214 1.3995043 0.48498031 -0.86961967 -0.058370713 -0.85965866 -1.2727696 0.45570874 0.73393697 0.2567143 1.4261572 -0.37773672 -0.7339463 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 1x1x32] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Squeeze(1, input_shape=(1, 1, 32))) + input = np.random.random([1, 1, 1, 32]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[[0.20585343, 0.47011701, 0.14553177, 0.93915599, 0.57234281, + 0.91631229, 0.32244256, 0.94243351, 0.86595631, 0.73916763, + 0.35898731, 0.65208275, 0.07935983, 0.89313423, 0.68601269, + 0.48919672, 0.28406399, 0.20962799, 0.88071757, 0.45501821, + 0.60931183, 0.46709718, 0.14218838, 0.42517758, 0.9149958 , + 0.0843243 , 0.27302307, 0.75281922, 0.3688931 , 0.86913729, + 0.89774196, 0.77838838]]]] + + Output is + + .. code-block:: python + + [[[0.20585343, 0.470117 , 0.14553176, 0.939156 , 0.5723428 , + 0.9163123 , 0.32244256, 0.94243354, 0.8659563 , 0.73916763, + 0.3589873 , 0.65208274, 0.07935983, 0.89313424, 0.6860127 , + 0.48919672, 0.284064 , 0.20962799, 0.8807176 , 0.45501822, + 0.6093118 , 0.46709716, 0.14218839, 0.42517757, 0.9149958 , + 0.0843243 , 0.27302307, 0.75281924, 0.36889312, 0.8691373 , + 0.897742 , 0.7783884 ]]] +``` + +--- +### BinaryThreshold +Threshold the input. + +If an input element is smaller than the threshold value, it will be replaced by 0; otherwise, it will be replaced by 1. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + BinaryThreshold(value = 1e-6, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + BinaryThreshold(value=1e-6, input_shape=None, name=None) +``` + +**Parameters:** + +* `value`: The threshold value to compare with. Default is 1e-6. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.BinaryThreshold + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(BinaryThreshold[Float](inputShape = Shape(2, 3, 4))) + val input = Tensor[Float](2, 2, 3, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + -1.1907398 -0.18995096 -2.0344417 -1.3789974 + -1.8801064 -0.74757665 -0.4339697 0.0058485097 + 0.7012256 -0.6363152 2.0156987 -0.5512639 + + (1,2,.,.) = + -0.5251603 0.082127444 0.29550993 1.6357868 + -1.3828015 -0.11842779 0.3316966 -0.14360528 + 0.21216457 -0.117370956 -0.12934707 -0.35854268 + + (2,1,.,.) = + -0.9071151 -2.8566089 -0.4796377 -0.915065 + -0.8439908 -0.25404388 -0.39926198 -0.15191565 + -1.0496653 -0.403675 -1.3591816 0.5311797 + + (2,2,.,.) = + 0.53509855 -0.08892822 1.2196561 -0.62759316 + -0.47476718 -0.43337926 -0.10406987 1.4035174 + -1.7120812 1.1328355 0.9219375 1.3813454 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 1.0 + 1.0 0.0 1.0 0.0 + + (1,2,.,.) = + 0.0 1.0 1.0 1.0 + 0.0 0.0 1.0 0.0 + 1.0 0.0 0.0 0.0 + + (2,1,.,.) = + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 1.0 + + (2,2,.,.) = + 1.0 0.0 1.0 0.0 + 0.0 0.0 0.0 1.0 + 0.0 1.0 1.0 1.0 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(BinaryThreshold(input_shape=(2, 3, 4))) + input = np.random.random([2, 2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[[0.30421481, 0.47800487, 0.54249411, 0.90109767], + [0.72650405, 0.53096719, 0.66346109, 0.0589329 ], + [0.12994731, 0.92181174, 0.43129874, 0.97306968]], + + [[0.3031087 , 0.20339982, 0.69034712, 0.40191 ], + [0.57517034, 0.30159448, 0.4801747 , 0.75175084], + [0.8599362 , 0.93523811, 0.34768628, 0.10840162]]], + + + [[[0.46102959, 0.33029002, 0.69340103, 0.32885719], + [0.84405147, 0.03421879, 0.68242578, 0.03560338], + [0.12244515, 0.3610654 , 0.01312785, 0.84485178]], + + [[0.73472287, 0.75707757, 0.77070527, 0.40863145], + [0.01137898, 0.82896826, 0.1498069 , 0.22309423], + [0.92737483, 0.36217222, 0.06679799, 0.33304362]]]]) + + Output is + + .. code-block:: python + + array([[[[1., 1., 1., 1.], + [1., 1., 1., 1.], + [1., 1., 1., 1.]], + + [[1., 1., 1., 1.], + [1., 1., 1., 1.], + [1., 1., 1., 1.]]], + + + [[[1., 1., 1., 1.], + [1., 1., 1., 1.], + [1., 1., 1., 1.]], + + [[1., 1., 1., 1.], + [1., 1., 1., 1.], + [1., 1., 1., 1.]]]], dtype=float32) +``` + +--- +### Sqrt +Applies an element-wise square root operation to the input. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Sqrt(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Sqrt(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Sqrt + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Sqrt[Float](inputShape = Shape(3))) + val input = Tensor[Float](2, 3).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + 0.6950394 0.5234307 1.7375475 + 0.25833175 0.02685826 -0.6046901 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + 0.8336902 0.7234851 1.3181607 + 0.50826347 0.16388491 NaN + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Sqrt(input_shape=(3, ))) + input = np.random.random([2, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[0.2484558 , 0.65280218, 0.35286984], + [0.19616094, 0.30966802, 0.82148169]] + + Output is + + .. code-block:: python + + [[0.4984534 , 0.80796176, 0.5940285 ], + [0.4429006 , 0.55647826, 0.9063563 ]] +``` + +--- +### Mul +Multiply a single scalar factor to the incoming data + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Mul(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Mul(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. +* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Mul + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Mul[Float](inputShape = Shape(3, 4))) + val input = Tensor[Float](2, 3, 4).randn() + val output = model.forward(input) + + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + -1.2316265 -2.008802 -1.3908259 -0.61135375 + -0.48992255 0.1786112 0.18872596 0.49621895 + -0.6931602 -0.919745 -0.09019699 -0.41218707 + + (2,.,.) = + -0.3135355 -0.4385771 -0.3317269 1.0412029 + -0.8859662 0.17758773 -0.73779273 -0.4445366 + 0.3921595 1.6923207 0.014470488 0.4044164 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + -0.59036994 -0.9629025 -0.6666808 -0.29304734 + -0.2348403 0.0856158 0.09046422 0.23785843 + -0.33226058 -0.44087213 -0.043235175 -0.19757845 + + (2,.,.) = + -0.15029064 -0.21022828 -0.15901053 0.49909195 + -0.42468053 0.0851252 -0.3536548 -0.21308492 + 0.18797839 0.81119984 0.006936308 0.19385365 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Mul(input_shape=(3, 4))) + input = np.random.random([2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[ 0.22607292, 0.59806062, 0.19428923, 0.22928606], + [ 0.13804536, 0.1615547 , 0.52824658, 0.52794904], + [ 0.4049169 , 0.94109084, 0.58158453, 0.78368633]], + + [[ 0.86233305, 0.47995805, 0.80430949, 0.9931171 ], + [ 0.35179631, 0.33615276, 0.87756877, 0.73560288], + [ 0.29775703, 0.11404466, 0.77695536, 0.97580018]]]) + + Output is + + .. code-block:: python + + array([[[-0.22486402, -0.59486258, -0.1932503 , -0.22805998], + [-0.13730718, -0.1606908 , -0.52542186, -0.52512592], + [-0.40275168, -0.93605846, -0.57847458, -0.77949566]], + + [[-0.85772187, -0.47739154, -0.80000854, -0.9878065 ], + [-0.34991512, -0.33435524, -0.87287611, -0.73166931], + [-0.29616481, -0.11343482, -0.77280068, -0.97058219]]], dtype=float32) +``` + +--- +### MulConstant +Multiply the input by a (non-learnable) scalar constant. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + MulConstant(constant, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + MulConstant(constant, input_shape=None, name=None) +``` + +**Parameters:** + +* `constant`: The scalar constant to be multiplied. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.MulConstant + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(MulConstant[Float](2.2, inputShape = Shape(3, 4))) + val input = Tensor[Float](2, 3, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + -0.16873977 1.0812985 1.0942211 -0.67091423 + 1.0086882 0.5915831 0.26184535 -1.361431 + 1.5616825 -0.037591368 1.2794676 1.0692137 + + (2,.,.) = + 0.29868057 -0.23266982 -0.7679556 -2.209848 + -0.13954644 -0.1368473 -0.54510623 1.8397199 + -0.58691734 -0.56410027 -1.5567777 0.050648995 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + -0.3712275 2.3788567 2.4072864 -1.4760114 + 2.219114 1.3014828 0.57605976 -2.9951482 + 3.4357016 -0.08270101 2.8148286 2.3522704 + + (2,.,.) = + 0.6570973 -0.5118736 -1.6895024 -4.8616657 + -0.3070022 -0.30106407 -1.1992338 4.047384 + -1.2912182 -1.2410206 -3.424911 0.11142779 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import * + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(MulConstant(2.2, input_shape=(3, 4))) + input = np.random.random([2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[0.39874191, 0.66634984, 0.23907766, 0.31587494], + [0.78842014, 0.93057835, 0.80739529, 0.71541279], + [0.2231424 , 0.3372844 , 0.94678072, 0.52928034]], + + [[0.60142458, 0.41221671, 0.00890549, 0.32069845], + [0.51122554, 0.76280426, 0.87579418, 0.17182832], + [0.54133184, 0.19814384, 0.92529327, 0.5616615 ]]] + + Output is + + .. code-block:: python + + [[[0.8772322 , 1.4659697 , 0.5259709 , 0.6949249 ], + [1.7345244 , 2.0472724 , 1.7762697 , 1.5739082 ], + [0.4909133 , 0.7420257 , 2.0829177 , 1.1644168 ]], + + [[1.3231341 , 0.9068768 , 0.01959208, 0.7055366 ], + [1.1246961 , 1.6781695 , 1.9267472 , 0.37802234], + [1.19093 , 0.43591645, 2.0356452 , 1.2356553 ]]] +``` + +--- +### Scale +Scale is the combination of CMul and CAdd. + +Computes the element-wise product of the input and weight, with the shape of the weight "expand" to match the shape of the input. + +Similarly, perform an expanded bias and perform an element-wise add. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Scale(size, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Scale(size, input_shape=None, name=None) +``` + +**Parameters:** + +* `size`: Size of the weight and bias. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Scale + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + var array = Array(1, 2) + model.add(Scale[Float](array, inputShape = Shape(3))) + val input = Tensor[Float](2, 3).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + -0.006399727 -0.06412822 -0.2334789 + 0.31029955 1.6557469 1.9614618 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + 0.09936619 0.57585865 0.20324506 + 0.38537437 -0.8598822 -1.0186496 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import Scale + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Scale((2, 1), input_shape=(3, ))) + input = np.random.random([2, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[0.7242994 , 0.77888884, 0.71470432], + [0.03058471, 0.00602764, 0.57513629]] + + Output is + + .. code-block:: python + + [[1.0946966 , 1.1255064 , 1.0892813 ], + [0.58151895, 0.5909191 , 0.37307182]] +``` + +--- +### Log +Applies a log transformation to the input. + + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Log(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Log(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Log + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Log[Float](inputShape = Shape(2, 4, 4))) + val input = Tensor[Float](1, 2, 4, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + 0.38405678 -0.5502389 -0.383079 -0.988537 + -0.6294056 -0.7838047 0.8747865 -1.0659786 + -2.2445498 -0.5488076 -0.42898977 0.6916364 + 1.6542299 -0.9966279 -0.38244298 1.6954672 + + (1,2,.,.) = + 0.43478605 -0.6678534 1.9530942 -0.5209587 + 0.12899925 0.20572199 2.0359943 0.55223215 + 0.65247816 0.8792108 -0.38860792 0.48663738 + -1.0084358 0.31141177 0.69208467 0.48385203 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2x4x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + -0.95696485 NaN NaN NaN + NaN NaN -0.13377543 NaN + NaN NaN NaN -0.36869493 + 0.5033356 NaN NaN 0.5279584 + + (1,2,.,.) = + -0.83290124 NaN 0.6694149 NaN + -2.0479486 -1.5812296 0.7109843 -0.5937868 + -0.4269776 -0.12873057 NaN -0.720236 + NaN -1.1666392 -0.36804697 -0.72597617 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 1x2x4x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import Log + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Log(input_shape=(2, 4, 4))) + input = np.random.random([1, 2, 4, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[[0.90127539, 0.9861594 , 0.04722941, 0.63719453], + [0.46529477, 0.81511804, 0.24435558, 0.45003562], + [0.15170845, 0.35157662, 0.0925214 , 0.63852947], + [0.27817508, 0.42572846, 0.44363004, 0.03536394]], + + [[0.65027784, 0.00429838, 0.07434429, 0.18653305], + [0.19659183, 0.66647529, 0.77821197, 0.65894478], + [0.28212032, 0.52307663, 0.09589939, 0.71547588], + [0.84344158, 0.25291738, 0.52145649, 0.82982377]]]] + + Output is: + + .. code-block:: python + + [[[[-0.10394441, -0.01393729, -3.0527387 , -0.45068032], + [-0.76508415, -0.20442237, -1.4091308 , -0.79842854], + [-1.8857948 , -1.0453277 , -2.3803153 , -0.44858742], + [-1.2795045 , -0.85395354, -0.8127643 , -3.3420627 ]], + + [[-0.43035555, -5.4495163 , -2.5990484 , -1.6791469 ], + [-1.6266255 , -0.4057522 , -0.25075635, -0.41711554], + [-1.2654216 , -0.64802724, -2.3444557 , -0.33480743], + [-0.1702646 , -1.3746924 , -0.6511295 , -0.1865419 ]]]] +``` + +--- +### Identity +Identity just return the input to output. + +It's useful in same parallel container to get an origin input. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Identity(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Identity(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.layers.Identity + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Identity[Float](inputShape = Shape(4, 4))) + val input = Tensor[Float](3, 4, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + 1.9601166 -0.86010313 0.0023731247 -0.81219757 + 1.1469674 -1.5375912 -1.5348053 -0.34829113 + -1.236773 -0.7183283 -0.89256984 0.8605067 + 0.7937664 0.52992857 -1.6157389 0.36134166 + + (2,.,.) = + -0.44434744 -0.23848957 -0.01632014 -0.58109635 + -0.19856784 -2.3421717 -0.5868049 -0.76775354 + 0.80254126 1.78778 -1.1835604 1.4489703 + 0.8731402 0.8906672 0.2800079 -0.6715317 + + (3,.,.) = + 1.4093032 2.358169 -1.4620789 1.1904576 + -0.18263042 -0.31869793 2.01061 1.2159953 + -0.5801479 1.2949371 -0.7510707 -1.0707517 + 0.30815956 -1.161963 -0.26964024 -0.4759499 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 3x4x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 1.9601166 -0.86010313 0.0023731247 -0.81219757 + 1.1469674 -1.5375912 -1.5348053 -0.34829113 + -1.236773 -0.7183283 -0.89256984 0.8605067 + 0.7937664 0.52992857 -1.6157389 0.36134166 + + (2,.,.) = + -0.44434744 -0.23848957 -0.01632014 -0.58109635 + -0.19856784 -2.3421717 -0.5868049 -0.76775354 + 0.80254126 1.78778 -1.1835604 1.4489703 + 0.8731402 0.8906672 0.2800079 -0.6715317 + + (3,.,.) = + 1.4093032 2.358169 -1.4620789 1.1904576 + -0.18263042 -0.31869793 2.01061 1.2159953 + -0.5801479 1.2949371 -0.7510707 -1.0707517 + 0.30815956 -1.161963 -0.26964024 -0.4759499 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 3x4x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import Identity + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Identity(input_shape=(4, 4))) + input = np.random.random([3, 4, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[0.36751123, 0.92287101, 0.73894405, 0.33699379], + [0.69405782, 0.9653215 , 0.2617223 , 0.68205229], + [0.71455325, 0.99419333, 0.90886495, 0.10232991], + [0.1644055 , 0.30013138, 0.98921154, 0.26803146]], + + [[0.35898357, 0.72067882, 0.13236563, 0.71935521], + [0.30865626, 0.71098844, 0.86718946, 0.12531168], + [0.84916882, 0.84221518, 0.52186664, 0.87239729], + [0.50637899, 0.10890469, 0.86832705, 0.93581179]], + + [[0.19640105, 0.09341008, 0.12043328, 0.09261859], + [0.66019486, 0.07251262, 0.80929761, 0.39094486], + [0.63027391, 0.39537796, 0.55578905, 0.53933265], + [0.13885559, 0.56695373, 0.17036027, 0.4577097 ]]] + + Output is + + .. code-block:: python + + [[[0.36751124, 0.922871 , 0.73894405, 0.33699378], + [0.6940578 , 0.9653215 , 0.2617223 , 0.6820523 ], + [0.71455324, 0.9941933 , 0.908865 , 0.10232991], + [0.1644055 , 0.30013138, 0.98921156, 0.26803148]], + + [[0.35898358, 0.7206788 , 0.13236563, 0.7193552 ], + [0.30865628, 0.71098846, 0.86718947, 0.12531169], + [0.84916884, 0.8422152 , 0.5218666 , 0.8723973 ], + [0.506379 , 0.10890469, 0.868327 , 0.9358118 ]], + + [[0.19640104, 0.09341008, 0.12043328, 0.09261858], + [0.6601949 , 0.07251262, 0.8092976 , 0.39094487], + [0.63027394, 0.39537796, 0.55578905, 0.5393326 ], + [0.13885559, 0.5669537 , 0.17036027, 0.4577097 ]]] +``` + +--- +### Select +Select an index of the input in the given dim and return the subset part. + +The batch dimension needs to be unchanged. + +For example, if input is: + +[[1, 2, 3], + [4, 5, 6]] + +Select(1, 1) will give output [2 5] + +Select(1, -1) will give output [3 6] + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Select(dim, index, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Select(dim, index, input_shape=None, name=None) +``` + +**Parameters:** + +* `dim`: The dimension to select. 0-based index. Cannot select the batch dimension. -1 means the last dimension of the input. +* `index`: The index of the dimension to be selected. 0-based index. -1 means the last dimension of the input. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Select + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Select[Float](1, 2, inputShape = Shape(3, 1, 3))) + val input = Tensor[Float](1, 3, 1, 3).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + -0.67646945 -0.5485965 -0.11103154 + (1,2,.,.) = + -0.13488655 0.43843046 -0.04482145 + (1,3,.,.) = + -0.18094881 0.19431554 -1.7624844 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 1x3x1x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + -0.18094881 0.19431554 -1.7624844 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 1x1x3] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import Select + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(Select(1, 2, input_shape=(3, 1, 3))) + input = np.random.random([1, 3, 1, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[[0.53306099, 0.95147881, 0.15222129]], + [[0.89604861, 0.90160974, 0.5230576 ]], + [[0.70779386, 0.14438568, 0.37601195]]]]) + + Output is: + + .. code-block:: python + + array([[[0.7077939 , 0.14438568, 0.37601194]]], dtype=float32) +``` + +--- +### Dense +A densely-connected NN layer. + +The most common input is 2D. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Dense(outputDim, init = "glorot_uniform", activation = null, wRegularizer = null, bRegularizer = null, bias = true, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Dense(output_dim, init="glorot_uniform", activation=None, W_regularizer=None, b_regularizer=None, bias=True, input_dim=None, input_shape=None, name=None) +``` + +**Parameters:** + +* `outputDim`: The size of the output dimension. +* `init`: Initialization method for the weights of the layer. Default is Xavier.You can also pass in corresponding string representations such as 'glorot_uniform' or 'normal', etc. for simple init methods in the factory method. +* `activation`: Activation function to use. Default is null.You can also pass in corresponding string representations such as 'relu'or 'sigmoid', etc. for simple activations in the factory method. +* `wRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the input weights matrices. Default is null. +* `bRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the bias. Default is null. +* `bias`: Whether to include a bias (i.e. make the layer affine rather than linear). Default is true. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Dense + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Dense[Float](5, activation = "relu", inputShape = Shape(4))) + val input = Tensor[Float](2, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + 1.4289935 -1.7659454 -0.08306135 -1.0153456 + 1.0191492 0.37392816 1.3076705 -0.19495767 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + 0.5421522 0.49008092 0.0 0.0 0.0 + 0.07940009 0.0 0.12953377 0.0 0.0 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x5] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import Dense + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(Dense(5, activation="relu", input_shape=(4, ))) + input = np.random.random([2, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[0.64593485, 0.67393322, 0.72505368, 0.04654095], + [0.19430753, 0.47800889, 0.00743648, 0.6412403 ]]) + + Output is + + .. code-block:: python + + array([[0. , 0. , 1.2698183 , 0. , 0.10656227], + [0. , 0. , 0.6236721 , 0.00299606, 0.29664695]], + dtype=float32) +``` + +--- +### Negative +Computes the negative value of each element of the input. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Negative(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Negative(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Negative + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Negative[Float](inputShape = Shape(2, 3))) + val input = Tensor[Float](2, 2, 3).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + 1.031705 -0.5723963 1.998631 + -0.32908052 2.4069138 -2.4111257 + (2,.,.) = + 0.5355049 -1.4404331 -0.38116863 + -0.45641592 -1.1485358 0.94766915 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + -1.031705 0.5723963 -1.998631 + 0.32908052 -2.4069138 2.4111257 + (2,.,.) = + -0.5355049 1.4404331 0.38116863 + 0.45641592 1.1485358 -0.94766915 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import Negative + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(Negative(input_shape=(2, 3))) + input = np.random.random([2, 2, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[0.39261261, 0.03164615, 0.32179116], + [0.11969367, 0.61610712, 0.42573733]], + [[0.36794656, 0.90912174, 0.540356 ], + [0.42667627, 0.04154093, 0.84692964]]]) + + Output is + + .. code-block:: python + + array([[[-0.3926126 , -0.03164615, -0.32179114], + [-0.11969367, -0.6161071 , -0.42573732]], + [[-0.36794657, -0.90912175, -0.540356 ], + [-0.42667627, -0.04154094, -0.84692967]]], dtype=float32) +``` + +--- +### CAdd + +This layer has a bias with given size. + +The bias will be added element-wise to the input. + +If the element number of the bias matches the input, a simple element-wise addition will be done. + +Or the bias will be expanded to the same size of the input. + +The expand means repeat on unmatched singleton dimension (if some unmatched dimension isn't a singleton dimension, an error will be raised). + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + CAdd(size, bRegularizer = null, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + CAdd(size, b_regularizer=None, input_shape=None, name=None) +``` + +**Parameters:** + +* `size`: the size of the bias +* `bRegularizer`: An instance of [Regularizer](https://bigdl-project.github.io/master/#APIGuide/Regularizers/), applied to the bias. Default is null. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.CAdd + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(CAdd[Float](Array(2, 3), inputShape = Shape(2, 3))) + val input = Tensor[Float](2, 2, 3).rand() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + 0.2183351 0.32434112 0.89350265 + 0.3348259 0.78677046 0.24054797 + (2,.,.) = + 0.9945844 0.72363794 0.7737936 + 0.05522544 0.3517818 0.7417069 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 0.1358028 0.6956667 1.0837181 + 0.6767027 0.7955346 0.5063505 + (2,.,.) = + 0.9120521 1.0949634 0.96400905 + 0.3971022 0.36054593 1.0075095 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import CAdd + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(CAdd([2, 1], input_shape=(2, 3))) + input = np.random.rand(2, 2, 3) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[0.4122004 , 0.73289359, 0.11500016], + [0.26974491, 0.32166632, 0.91408442]], + [[0.66824327, 0.80271314, 0.75981145], + [0.39271431, 0.07312566, 0.4966805 ]]]) + + Output is + + .. code-block:: python + + array([[[ 0.06560206, 0.38629526, -0.23159817], + [ 0.44287407, 0.4947955 , 1.0872136 ]], + [[ 0.32164496, 0.45611483, 0.41321313], + [ 0.56584346, 0.24625483, 0.6698097 ]]], dtype=float32) +``` + +--- +### RepeatVector +Repeats the input n times. + +The input of this layer should be 2D, i.e. (num_samples, features). +The output of thi layer should be 3D, i.e. (num_samples, n, features). + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + RepeatVector(n, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + RepeatVector(n, input_shape=None, name=None) +``` + +**Parameters:** + +* `n`: Repetition factor. Integer. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. +* `name`: String to set the name of the layer. If not specified, its name will by default to be a generated string. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.RepeatVector + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(RepeatVector[Float](4, inputShape = Shape(3))) + val input = Tensor[Float](2, 3).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + -0.31839952 -0.3495366 0.542486 + -0.54981124 -0.8428188 0.8225184 + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + -0.31839952 -0.3495366 0.542486 + -0.31839952 -0.3495366 0.542486 + -0.31839952 -0.3495366 0.542486 + -0.31839952 -0.3495366 0.542486 + + (2,.,.) = + -0.54981124 -0.8428188 0.8225184 + -0.54981124 -0.8428188 0.8225184 + -0.54981124 -0.8428188 0.8225184 + -0.54981124 -0.8428188 0.8225184 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x4x3] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.layers import RepeatVector + from bigdl.dllib.keras.models import Sequential + + model = Sequential() + model.add(RepeatVector(4, input_shape=(3, ))) + input = np.random.random([2, 3]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[ 0.90715922, 0.54594769, 0.53952404], + [ 0.08989831, 0.07265549, 0.45830114]]) + + Output is + + .. code-block:: python + + array([[[ 0.90715921, 0.54594767, 0.53952402], + [ 0.90715921, 0.54594767, 0.53952402], + [ 0.90715921, 0.54594767, 0.53952402], + [ 0.90715921, 0.54594767, 0.53952402]], + + [[ 0.08989831, 0.07265549, 0.45830116], + [ 0.08989831, 0.07265549, 0.45830116], + [ 0.08989831, 0.07265549, 0.45830116], + [ 0.08989831, 0.07265549, 0.45830116]]], dtype=float32) +``` +--- +### GaussianSampler +Takes {mean, log_variance} as input and samples from the Gaussian distribution. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + GaussianSampler(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + GaussianSampler(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object that consists of two identical Single Shape. For Python API, it should be a list of two identical shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.GaussianSampler + import com.intel.analytics.bigdl.utils.{Shape, MultiShape, T} + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + val shape1 = Shape(2, 3) + val shape2 = Shape(2, 3) + model.add(GaussianSampler[Float](inputShape = MultiShape(List(shape1,shape2)))) + val input1 = Tensor[Float](2, 2, 3).rand(0, 1) + val input2 = Tensor[Float](2, 2, 3).rand(0, 1) + val input = T(1 -> input1, 2 -> input2) + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.utils.Table = + { + 2: (1,.,.) = + 0.9996127 0.8964211 0.7424038 + 0.40628982 0.37035564 0.20108517 + + (2,.,.) = + 0.6974727 0.60202897 0.1535999 + 0.012422224 0.5993025 0.96206 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + 1: (1,.,.) = + 0.21060324 0.576583 0.21633287 + 0.1484059 0.2730577 0.25317845 + + (2,.,.) = + 0.58513683 0.58095694 0.18811373 + 0.7029449 0.41235915 0.44636542 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + } + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 1.5258198 1.9536011 -1.8591263 + -1.0618867 -0.751225 0.35412917 + + (2,.,.) = + 1.3334517 -0.60312974 0.7324476 + 0.09502721 0.8094909 0.44807082 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.models import Sequential + from bigdl.dllib.keras.layers import GaussianSampler + + model = Sequential() + model.add(GaussianSampler(input_shape=[(3,),(3,)])) + input1 = np.random.random([2, 3]) + input2 = np.random.random([2, 3]) + input = [input1, input2] + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[0.79941342, 0.87462822, 0.9516901 ], + [0.20111287, 0.54634077, 0.83614511]], + + [[0.31886989, 0.22829382, 0.84355419], + [0.51186641, 0.28043938, 0.29440057]]] + + Output is + + .. code-block:: python + + [[ 0.71405387 2.2944303 -0.41778684] + [ 0.84234 2.3337283 -0.18952972]] +``` + +--- +### Exp +Applies element-wise exp to the input. + +When you use this layer as the first layer of a model, you need to provide the argument inputShape (a Single Shape, does not include the batch dimension). + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Exp(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Exp(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object that consists of two identical Single Shape. For Python API, it should be a list of two identical shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Exp + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Exp[Float](inputShape = Shape(2, 3, 4))) + val input = Tensor[Float](2, 2, 3, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + -1.5841372 -0.13795324 -2.144475 0.09272669 + 1.055668 -1.2310301 1.2145554 -0.6073714 + 0.9296467 0.2923885 1.3364213 0.1652137 + + (1,2,.,.) = + 0.2099718 -0.3856573 -0.92586 -0.5317779 + 0.6618383 -0.9677452 -1.5014665 -0.35464883 + 2.045924 -0.317644 -1.812726 0.95438373 + + (2,1,.,.) = + -0.4536791 -0.34785584 1.6424289 -0.07981159 + -0.8022624 -0.4211059 0.3461831 1.9598864 + -0.84695745 -0.6115283 0.7729755 2.3077402 + + (2,2,.,.) = + -0.08438411 -0.908458 0.6688936 -0.7292123 + -0.26337254 0.55425745 -0.14925817 -0.010179609 + -0.62562865 -1.0517743 -0.23839666 -1.144982 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.20512469 0.8711394 0.11712951 1.0971619 + 2.8738942 0.29199165 3.3687959 0.544781 + 2.533614 1.3396233 3.8054006 1.1796452 + + (1,2,.,.) = + 1.2336433 0.6800035 0.39619055 0.5875594 + 1.9383523 0.37993878 0.22280318 0.7014197 + 7.7363033 0.7278619 0.16320862 2.5970695 + + (2,1,.,.) = + 0.63528657 0.70620066 5.167706 0.92329025 + 0.44831353 0.6563206 1.4136615 7.0985208 + 0.42871734 0.5425211 2.1662023 10.051684 + + (2,2,.,.) = + 0.9190782 0.4031454 1.9520763 0.48228875 + 0.76845556 1.740648 0.8613467 0.98987204 + 0.53492504 0.34931743 0.7878901 0.31822965 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.models import Sequential + from bigdl.dllib.keras.layers import Exp + + model = Sequential() + model.add(Exp(input_shape=(2, 3, 4))) + input = np.random.random([2, 2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[[0.93104587 0.94000338 0.84870765 0.98645553] + [0.83708846 0.33375541 0.50119834 0.24879265] + [0.51966475 0.84514791 0.15496452 0.61538968]] + + [[0.57250337 0.42520832 0.94850757 0.54317573] + [0.64228691 0.9904079 0.01008592 0.51365217] + [0.78640595 0.7717037 0.51277595 0.24245034]]] + + + [[[0.82184752 0.92537331 0.20632728 0.47539445] + [0.44604637 0.1507692 0.5437313 0.2074501 ] + [0.93661363 0.93962609 0.29230559 0.74850958]] + + [[0.11659768 0.76177132 0.33194573 0.20695088] + [0.49636212 0.85987328 0.49767861 0.96774006] + [0.67669121 0.15542122 0.69981032 0.3349874 ]]]] + + Output is + + .. code-block:: python + + [[[[2.5371614 2.5599902 2.3366253 2.6817122] + [2.3096325 1.3962016 1.6506982 1.2824761] + [1.6814638 2.3283222 1.1676165 1.8503776]] + + [[1.7726992 1.5299091 2.5818534 1.721465 ] + [1.9008229 2.6923325 1.010137 1.6713842] + [2.1954916 2.163449 1.6699204 1.2743679]]] + + + [[[2.2746985 2.52281 1.2291554 1.6086487] + [1.5621239 1.1627283 1.7224218 1.2305363] + [2.551327 2.5590243 1.3395122 2.1138473]] + + [[1.1236672 2.1420672 1.3936772 1.2299222] + [1.6427343 2.3628614 1.6448984 2.6319895] + [1.9673574 1.16815 2.0133708 1.3979228]]]] + +``` + +--- +### Square +Applies an element-wise square operation to the input. + +When you use this layer as the first layer of a model, you need to provide the argument inputShape (a Single Shape, does not include the batch dimension). + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Square(inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Square(input_shape=None, name=None) +``` + +**Parameters:** + +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`MultiShape`](../keras-api-scala/#shape) object that consists of two identical Single Shape. For Python API, it should be a list of two identical shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Square + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Square[Float](inputShape = Shape(2, 3, 4))) + val input = Tensor[Float](2, 2, 3, 4).randn() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + -0.108013034 1.8879265 1.2232096 -1.5076439 + 1.4895755 -0.37966672 -0.34892964 0.15224025 + -0.9296686 -1.1523775 0.14153497 -0.26954007 + + (1,2,.,.) = + -1.0875931 2.190617 -0.6903083 1.0039362 + -0.1275677 -1.1096588 0.37359753 -0.17367937 + 0.23349741 0.14639114 -0.2330162 0.5343827 + + (2,1,.,.) = + 0.3222191 0.21463287 -1.0157064 -0.22627507 + 1.1714277 0.43371263 1.069315 0.5122436 + 0.1958086 -1.4601041 2.5394423 -0.470833 + + (2,2,.,.) = + -0.38708544 -0.951611 -0.37234613 0.26813275 + 1.9477026 0.32779223 -1.2308712 -2.2376378 + 0.19652915 0.3304719 -1.7674786 -0.86961496 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.011666816 3.5642662 1.4962418 2.2729902 + 2.218835 0.14414681 0.1217519 0.023177093 + 0.86428374 1.3279738 0.020032147 0.07265185 + + (1,2,.,.) = + 1.1828587 4.7988033 0.47652552 1.0078878 + 0.016273517 1.2313428 0.13957511 0.030164523 + 0.05452104 0.021430366 0.054296546 0.28556487 + + (2,1,.,.) = + 0.10382515 0.046067268 1.0316595 0.05120041 + 1.3722429 0.18810664 1.1434345 0.26239353 + 0.038341008 2.131904 6.448767 0.22168371 + + (2,2,.,.) = + 0.14983514 0.9055635 0.13864164 0.07189517 + 3.7935455 0.10744774 1.5150439 5.007023 + 0.038623706 0.109211676 3.1239805 0.7562302 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + + .. tab:: Python Example + + .. code-block:: python + + import numpy as np + from bigdl.dllib.keras.models import Sequential + from bigdl.dllib.keras.layers import Square + + model = Sequential() + model.add(Square(input_shape=(2, 3, 4))) + input = np.random.random([2, 2, 3, 4]) + output = model.forward(input) + + Input is: + + .. code-block:: python + + [[[[0.8708819 0.2698243 0.55854849 0.71699472] + [0.66647234 0.72310216 0.8082119 0.66566951] + [0.6714764 0.61394108 0.35063125 0.60473593]] + + [[0.37993365 0.64222557 0.96762005 0.18931697] + [0.00529722 0.99133455 0.09786619 0.28988077] + [0.60052911 0.83712995 0.59847519 0.54361243]]] + + + [[[0.32832672 0.83316023 0.41272485 0.01963383] + [0.89593955 0.73433713 0.67529323 0.69711912] + [0.81251711 0.56755577 0.31958151 0.09795917]] + + [[0.46465895 0.22818875 0.31505317 0.41912166] + [0.87865447 0.3799063 0.091204 0.68144165] + [0.88274284 0.70479132 0.32074672 0.71771481]]]] + + Output is + + .. code-block:: python + + [[[[7.5843531e-01 7.2805151e-02 3.1197643e-01 5.1408142e-01] + [4.4418535e-01 5.2287674e-01 6.5320653e-01 4.4311589e-01] + [4.5088059e-01 3.7692365e-01 1.2294226e-01 3.6570552e-01]] + + [[1.4434958e-01 4.1245368e-01 9.3628860e-01 3.5840917e-02] + [2.8060573e-05 9.8274422e-01 9.5777912e-03 8.4030852e-02] + [3.6063525e-01 7.0078653e-01 3.5817260e-01 2.9551446e-01]]] + + + [[[1.0779844e-01 6.9415593e-01 1.7034180e-01 3.8548734e-04] + [8.0270761e-01 5.3925103e-01 4.5602092e-01 4.8597506e-01] + [6.6018403e-01 3.2211956e-01 1.0213234e-01 9.5959986e-03]] + + [[2.1590793e-01 5.2070107e-02 9.9258497e-02 1.7566296e-01] + [7.7203369e-01 1.4432879e-01 8.3181690e-03 4.6436274e-01] + [7.7923489e-01 4.9673077e-01 1.0287846e-01 5.1511449e-01]]]] + +``` + +--- +### Power +Applies an element-wise power operation with scale and shift to the input. + +f(x) = (shift + scale * x)^power^ + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Power(power, scale = 1, shift = 0, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Power(power, scale=1, shift=0, input_shape=None, name=None) +``` + +**Parameters:** + +* `power`: The exponent +* `scale`: The scale parameter. Default is 1. +* `shift`: The shift parameter. Default is 0. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Power + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Power[Float](2, inputShape = Shape(2, 3))) + val input = Tensor[Float](2, 2, 3).rand() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + 0.24691099 0.7588585 0.5785183 + 0.10356348 0.2252714 0.3129436 + + (2,.,.) = + 0.6277785 0.75136995 0.044648796 + 0.46396527 0.9793776 0.92727077 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 0.060965035 0.5758662 0.3346834 + 0.010725395 0.050747205 0.0979337 + + (2,.,.) = + 0.39410582 0.5645568 0.001993515 + 0.21526377 0.95918053 0.8598311 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import Power + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(Power(2, input_shape=(2, 3))) + input = np.random.rand(2, 2, 3) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[0.5300817 , 0.18128031, 0.19534253], + [0.28380639, 0.78365165, 0.6893 ]], + + [[0.05574091, 0.400077 , 0.77051193], + [0.033559 , 0.61051396, 0.13970227]]]) + + Output is + + .. code-block:: python + + array([[[0.2809866 , 0.03286255, 0.03815871], + [0.08054607, 0.61410993, 0.4751345 ]], + + [[0.00310705, 0.16006161, 0.5936886 ], + [0.00112621, 0.37272733, 0.01951673]]], dtype=float32) +``` + +--- +### AddConstant +Add a (non-learnable) scalar constant to the input. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + AddConstant(constant, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + AddConstant(constant, input_shape=None, name=None) +``` + +**Parameters:** + +* `constant`: The scalar constant to be added. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.AddConstant + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(AddConstant[Float](1, inputShape = Shape(2, 3))) + val input = Tensor[Float](2, 2, 3).rand() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,.,.) = + 0.5658301 0.3508225 0.4012322 + 0.1941942 0.18934165 0.6909284 + + (2,.,.) = + 0.5985211 0.5485885 0.778548 + 0.16745302 0.10363362 0.92185616 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,.,.) = + 1.5658301 1.3508224 1.4012322 + 1.1941942 1.1893417 1.6909285 + + (2,.,.) = + 1.5985211 1.5485885 1.778548 + 1.167453 1.1036336 1.9218562 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import AddConstant + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(AddConstant(1, input_shape=(2, 3))) + input = np.random.rand(2, 2, 3) + output = model.forward(input) + + + Input is: + + .. code-block:: python + + array([[[0.71730919, 0.07752598, 0.10448237], + [0.52319608, 0.38668494, 0.19588814]], + + [[0.15496092, 0.48405899, 0.41441248], + [0.13792111, 0.7523953 , 0.55991187]]]) + + Output is + + .. code-block:: python + + array([[[1.7173092, 1.077526 , 1.1044824], + [1.5231961, 1.3866849, 1.1958882]], + + [[1.1549609, 1.484059 , 1.4144125], + [1.1379211, 1.7523953, 1.5599118]]], dtype=float32) +``` + +--- +### Narrow +Narrow the input with the number of dimensions not being reduced. + +The batch dimension needs to be unchanged. + +For example, if input is: + +[[1 2 3], + [4 5 6]] + +Narrow(1, 1, 2) will give output + +[[2 3], + [5 6]] + +Narrow(1, 2, -1) will give output + +[3, + 6] + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Narrow(dim, offset, length = 1, inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + Narrow(dim, offset, length=1, input_shape=None, name=None) +``` + +**Parameters:** + +* `dim`: The dimension to narrow. 0-based index. Cannot narrow the batch dimension. + -1 means the last dimension of the input. +* `offset`: Non-negative integer. The start index on the given dimension. 0-based index. +* `length`: The length to narrow. Default is 1. + Can use a negative length such as -1 in the case where input size is unknown. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Narrow + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Narrow[Float](1, 1, inputShape = Shape(2, 3, 4))) + val input = Tensor[Float](2, 2, 3, 4).rand() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + 0.13770224 0.63719153 0.7776689 0.46612367 + 0.9026256 0.11982094 0.8282868 0.05095969 + 0.889799 0.6386537 0.35438475 0.298043 + + (1,2,.,.) = + 0.5029727 0.20103335 0.20150806 0.06437344 + 0.2255908 0.5388977 0.59737855 0.5210477 + 0.4055072 0.11848069 0.7118382 0.9796308 + + (2,1,.,.) = + 0.63957494 0.1921936 0.7749439 0.19744827 + 0.91683346 0.16140814 0.9753973 0.8161283 + 0.8481694 0.8802563 0.1233245 0.5732614 + + (2,2,.,.) = + 0.275001 0.35905758 0.15939762 0.09233412 + 0.16610192 0.032060683 0.37298614 0.48936844 + 0.031097537 0.82767457 0.10246291 0.9951448 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x3x4] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.5029727 0.20103335 0.20150806 0.06437344 + 0.2255908 0.5388977 0.59737855 0.5210477 + 0.4055072 0.11848069 0.7118382 0.9796308 + + (2,1,.,.) = + 0.275001 0.35905758 0.15939762 0.09233412 + 0.16610192 0.032060683 0.37298614 0.48936844 + 0.031097537 0.82767457 0.10246291 0.9951448 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x1x3x4] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import Narrow + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(Narrow(1, 1, input_shape=(2, 3, 4))) + input = np.random.rand(2, 2, 3, 4) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[[0.74305305, 0.33925069, 0.31289333, 0.43703923], + [0.28316902, 0.3004414 , 0.40298034, 0.37476436], + [0.18825825, 0.38979411, 0.32963262, 0.37783457]], + + [[0.14824117, 0.43532988, 0.57077087, 0.91535978], + [0.46375725, 0.90511296, 0.18859044, 0.92820822], + [0.13675737, 0.48270908, 0.04260755, 0.97255687]]], + [[[0.4836805 , 0.45262542, 0.7233705 , 0.63486529], + [0.07472717, 0.5715716 , 0.57029986, 0.26475783], + [0.56757079, 0.27602746, 0.45799196, 0.74420842]], + + [[0.89048761, 0.08280716, 0.99030481, 0.35956427], + [0.70802689, 0.14425212, 0.08320864, 0.82271697], + [0.6915224 , 0.70490768, 0.41218963, 0.37024863]]]]) + + Output is + + .. code-block:: python + + array([[[[0.14824118, 0.43532988, 0.57077086, 0.9153598 ], + [0.46375725, 0.905113 , 0.18859044, 0.92820823], + [0.13675737, 0.48270908, 0.04260755, 0.9725569 ]]], + + [[[0.8904876 , 0.08280716, 0.9903048 , 0.35956427], + [0.7080269 , 0.14425212, 0.08320864, 0.82271695], + [0.6915224 , 0.70490766, 0.41218963, 0.37024862]]]], + dtype=float32) +``` + +--- +### Permute +Permutes the dimensions of the input according to a given pattern. + +Useful for connecting RNNs and convnets together. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + Permute(dims, inputShape = null) + + + .. tab:: Python API + + .. code-block:: python + + Permute(dims, input_shape=None, name=None) +``` + +**Parameters:** + +* `dims`: Int array. Permutation pattern, does not include the batch dimension. + Indexing starts at 1. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.Permute + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential[Float]() + model.add(Permute[Float](Array(2, 1, 3), inputShape = Shape(2, 2, 3))) + val input = Tensor[Float](2, 2, 2, 3).rand() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + 0.8451549 0.06361471 0.7324815 + 0.31086245 0.21210302 0.35112163 + + (1,2,.,.) = + 0.61466074 0.50173014 0.8759959 + 0.19090249 0.671227 0.73089105 + (2,1,.,.) = + 0.47867084 0.9341955 0.063592255 + 0.24063066 0.502274 0.9114748 + (2,2,.,.) = + 0.93335986 0.25173688 0.88615775 + 0.5394321 0.330763 0.89036304 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x2x3] + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.8451549 0.06361471 0.7324815 + 0.61466074 0.50173014 0.8759959 + + (1,2,.,.) = + 0.31086245 0.21210302 0.35112163 + 0.19090249 0.671227 0.73089105 + (2,1,.,.) = + 0.47867084 0.9341955 0.063592255 + 0.93335986 0.25173688 0.88615775 + (2,2,.,.) = + 0.24063066 0.502274 0.9114748 + 0.5394321 0.330763 0.89036304 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x2x3] + + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import Permute + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(Permute((2, 1, 3), input_shape=(2, 2, 3))) + input = np.random.rand(2, 2, 2, 3) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[[0.14016896, 0.7275626 , 0.79087092], + [0.57259566, 0.97387138, 0.70001999]], + + [[0.9232002 , 0.07644555, 0.24705828], + [0.17257354, 0.93951155, 0.46183983]]], + [[[0.79432476, 0.64299062, 0.33959594], + [0.58608318, 0.338014 , 0.92602687]], + + [[0.32638575, 0.69032582, 0.25168083], + [0.46813027, 0.95118373, 0.13145026]]]]) + + Output is: + + .. code-block:: python + + array([[[[0.14016896, 0.7275626 , 0.7908709 ], + [0.9232002 , 0.07644555, 0.24705827]], + + [[0.57259566, 0.97387135, 0.70002 ], + [0.17257354, 0.93951154, 0.46183982]]], + [[[0.79432476, 0.64299065, 0.33959594], + [0.32638577, 0.6903258 , 0.25168082]], + [[0.5860832 , 0.338014 , 0.9260269 ], + [0.46813026, 0.95118374, 0.13145027]]]], dtype=float32) + +``` +--- +### ResizeBilinear +Resize the input image with bilinear interpolation. The input image must be a float tensor with NHWC or NCHW layout. + +```eval_rst +.. tabs:: + + .. tab:: Scala API + + .. code-block:: scala + + ResizeBilinear(outputHeight, outputWidth, alignCorners = false, dimOrdering = "th", inputShape = null) + + .. tab:: Python API + + .. code-block:: python + + ResizeBilinear(output_height, output_width, align_corner=False, dim_ordering="th", input_shape=(2, 3, 5, 7), name=None) +``` + +**Parameters:** + +* `outputHeight`: output height +* `outputWidth`: output width +* `alignCorners`: align corner or not +* `dimOrdering`: Format of input data. Either DataFormat.NCHW (dimOrdering='th') or DataFormat.NHWC (dimOrdering='tf'). Default is NCHW. +* `inputShape`: Only need to specify this argument when you use this layer as the first layer of a model. For Scala API, it should be a [`Shape`](../keras-api-scala/#shape) object. For Python API, it should be a shape tuple. Batch dimension should be excluded. + +```eval_rst +.. tabs:: + + .. tab:: Scala Example + + .. code-block:: scala + + import com.intel.analytics.bigdl.dllib.keras.models.Sequential + import com.intel.analytics.bigdl.dllib.keras.layers.ResizeBilinear + import com.intel.analytics.bigdl.dllib.utils.Shape + import com.intel.analytics.bigdl.tensor.Tensor + + val model = Sequential() + model.add(ResizeBilinear[Float](2, 3, inputShape = Shape(2, 3, 5))) + val input = Tensor[Float](2, 2, 3, 5).rand() + val output = model.forward(input) + + Input is: + + .. code-block:: scala + + input: com.intel.analytics.bigdl.tensor.Tensor[Float] = + (1,1,.,.) = + 0.6991891 0.007127314 0.73871046 0.95916307 0.9433856 + 0.41275907 0.37573513 0.99193203 0.06930728 0.5922364 + 0.024281504 0.2592453 0.3898136 0.6635241 0.85888565 + + (1,2,.,.) = + 0.38028112 0.43709648 0.62538666 0.8468501 0.6445014 + 0.45252413 0.48801896 0.59471387 0.013207023 0.3567462 + 0.85187584 0.49279585 0.7973665 0.81287366 0.07852263 + + (2,1,.,.) = + 0.1452374 0.6140467 0.36384684 0.066476084 0.96101314 + 0.54862195 0.66091377 0.86857307 0.6844842 0.7368217 + 0.25342992 0.71737933 0.12789607 0.21691357 0.7543404 + + (2,2,.,.) = + 0.79176855 0.1204049 0.58971256 0.115073755 0.10459962 + 0.5225398 0.742363 0.7612815 0.9881919 0.13359445 + 0.9026869 0.13972941 0.92064524 0.9435532 0.5502235 + + [com.intel.analytics.bigdl.tensor.DenseTensor of... + + Output is: + + .. code-block:: scala + + output: com.intel.analytics.bigdl.nn.abstractnn.Activity = + (1,1,.,.) = + 0.6991891 0.4948494 0.9539039 + 0.21852028 0.5664119 0.48613077 + + (1,2,.,.) = + 0.38028112 0.56262326 0.7794005 + 0.6522 0.6274959 0.34790504 + + (2,1,.,.) = + 0.1452374 0.4472468 0.36465502 + 0.40102595 0.5618719 0.54899293 + + (2,2,.,.) = + 0.79176855 0.43327665 0.111582376 + 0.71261334 0.70765764 0.75788474 + + [com.intel.analytics.bigdl.tensor.DenseTensor of size 2x2x2x3] + + + .. tab:: Python Example + + .. code-block:: python + + from bigdl.dllib.keras.layers import ResizeBilinear + from bigdl.dllib.keras.models import Sequential + import numpy as np + + model = Sequential() + model.add(ResizeBilinear(2, 3, input_shape=(2, 3, 5, 5))) + input = np.random.rand(2, 2, 3, 5, 5) + output = model.forward(input) + + Input is: + + .. code-block:: python + + array([[[[0.43790358, 0.41882914, 0.71929122, 0.19673119, 0.36950189], + [0.38808651, 0.34287751, 0.34076998, 0.02581254, 0.42406155], + [0.84648848, 0.18411068, 0.97545126, 0.5468195 , 0.32136674]], + + [[0.32965599, 0.06883324, 0.17350748, 0.01181338, 0.59180775], + [0.24667588, 0.36422516, 0.59648387, 0.48699443, 0.32323264], + [0.67661373, 0.58779956, 0.55286771, 0.59629101, 0.69727522]]], + + + [[[0.09462238, 0.35658325, 0.6787812 , 0.78676645, 0.99019452], + [0.81501527, 0.13348641, 0.71749101, 0.40543351, 0.3959018 ], + [0.608378 , 0.10531177, 0.78000335, 0.51679768, 0.65067605]], + + [[0.12074634, 0.92682843, 0.52227042, 0.98856558, 0.28105255], + [0.78411841, 0.19625097, 0.83108171, 0.03777509, 0.15700493], + [0.95528158, 0.94003855, 0.61092905, 0.68651048, 0.57563719]]]]) + + Output is + + .. code-block:: python + + array([[[[0.43790358, 0.61913717, 0.2543214 ], + [0.6172875 , 0.52657175, 0.3151154 ]], + + [[0.329656 , 0.13861606, 0.20514478], + [0.46164483, 0.541788 , 0.5311798 ]]], + + + [[[0.09462238, 0.57138187, 0.8545758 ], + [0.7116966 , 0.5389645 , 0.48184 ]], + + [[0.12074634, 0.6571231 , 0.752728 ], + [0.86969995, 0.6700518 , 0.36353552]]]], dtype=float32) + +``` +--- diff --git a/docs/readthedocs/source/doc/DLlib/Overview/freeze.md b/docs/readthedocs/source/doc/PythonAPI/DLlib/freeze.md similarity index 99% rename from docs/readthedocs/source/doc/DLlib/Overview/freeze.md rename to docs/readthedocs/source/doc/PythonAPI/DLlib/freeze.md index fbe1b6b3..28577490 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/freeze.md +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/freeze.md @@ -1,4 +1,5 @@ -## Model Freeze +# Model Freeze + To "freeze" a model means to exclude some layers of model from training. ```scala diff --git a/docs/readthedocs/source/doc/PythonAPI/DLlib/index.rst b/docs/readthedocs/source/doc/PythonAPI/DLlib/index.rst new file mode 100644 index 00000000..df15fdd0 --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/index.rst @@ -0,0 +1,13 @@ +DLlib API +================== + +.. toctree:: + :maxdepth: 1 + + model.rst + core_layers.md + optim-Methods.md + regularizers.md + learningrate-Scheduler.md + freeze.md + clipping.md \ No newline at end of file diff --git a/docs/readthedocs/source/doc/DLlib/Overview/learningrate-Scheduler.md b/docs/readthedocs/source/doc/PythonAPI/DLlib/learningrate-Scheduler.md similarity index 99% rename from docs/readthedocs/source/doc/DLlib/Overview/learningrate-Scheduler.md rename to docs/readthedocs/source/doc/PythonAPI/DLlib/learningrate-Scheduler.md index c714b42e..06e1a3e5 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/learningrate-Scheduler.md +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/learningrate-Scheduler.md @@ -1,3 +1,8 @@ +# Learning Rate Scheduler + +-------- + + ## Poly ## **Scala:** diff --git a/docs/readthedocs/source/doc/PythonAPI/DLlib/model.rst b/docs/readthedocs/source/doc/PythonAPI/DLlib/model.rst new file mode 100644 index 00000000..0a7c2182 --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/model.rst @@ -0,0 +1,17 @@ +Model/Sequential +================== + +dllib.keras.models.Model +--------------------------- + +.. autoclass:: bigdl.dllib.keras.models.Model + :members: + :undoc-members: + + +dllib.keras.models.Sequential +--------------------------- + +.. autoclass:: bigdl.dllib.keras.models.Sequential + :members: + :undoc-members: diff --git a/docs/readthedocs/source/doc/DLlib/Overview/optim-Methods.md b/docs/readthedocs/source/doc/PythonAPI/DLlib/optim-Methods.md similarity index 98% rename from docs/readthedocs/source/doc/DLlib/Overview/optim-Methods.md rename to docs/readthedocs/source/doc/PythonAPI/DLlib/optim-Methods.md index 0330e5fb..305e085d 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/optim-Methods.md +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/optim-Methods.md @@ -1,3 +1,7 @@ +# Optimizer + +-------- + ## Adam ## **Scala:** @@ -11,16 +15,16 @@ optim = Adam(learningrate=1e-3, learningrate_decay=0.0, beta1=0.9, beta2=0.999, An implementation of Adam optimization, first-order gradient-based optimization of stochastic objective functions. http://arxiv.org/pdf/1412.6980.pdf - `learningRate` learning rate. Default value is 1e-3. - + `learningRate` learning rate. Default value is 1e-3. + `learningRateDecay` learning rate decay. Default value is 0.0. - + `beta1` first moment coefficient. Default value is 0.9. - + `beta2` second moment coefficient. Default value is 0.999. - + `Epsilon` for numerical stability. Default value is 1e-8. - + **Scala example:** ```scala @@ -66,7 +70,7 @@ def rosenBrock(x: Tensor[Float]): (Float, Tensor[Float]) = { dxout.narrow(1, 2, d - 1).add(x0) (fout, dxout) - } + } val x = Tensor(2).fill(0) > print(optm.optimize(rosenBrock, x)) (0.0019999996 @@ -76,7 +80,7 @@ val x = Tensor(2).fill(0) **Python example:** ```python optim_method = Adam(learningrate=0.002) - + optimizer = Optimizer( model=mlp_model, training_rdd=train_data, @@ -104,10 +108,10 @@ optim_method = SGD(learningrate=1e-3,learningrate_decay=0.0,weightdecay=0.0, weightdecays=None,bigdl_type="float") ``` -A plain implementation of SGD which provides optimize method. After setting -optimization method when create Optimize, Optimize will call optimization method at the end of +A plain implementation of SGD which provides optimize method. After setting +optimization method when create Optimize, Optimize will call optimization method at the end of each iteration. - + **Scala example:** ```scala val optimMethod = new SGD[Float](learningRate= 1e-3,learningRateDecay=0.0, @@ -123,7 +127,7 @@ optim_method = SGD(learningrate=1e-3,learningrate_decay=0.0,weightdecay=0.0, momentum=0.0,dampening=DOUBLEMAX,nesterov=False, leaningrate_schedule=None,learningrates=None, weightdecays=None,bigdl_type="float") - + optimizer = Optimizer( model=mlp_model, training_rdd=train_data, @@ -136,7 +140,7 @@ optimizer = Optimizer( ## Adadelta ## -*AdaDelta* implementation for *SGD* +*AdaDelta* implementation for *SGD* It has been proposed in `ADADELTA: An Adaptive Learning Rate Method`. http://arxiv.org/abs/1212.5701. @@ -302,7 +306,7 @@ optimizer.setOptimMethod(optimMethod) optim_method = LBFGS(max_iter=20, max_eval=DOUBLEMAX, \ tol_fun=1e-5, tol_x=1e-9, n_correction=100, \ learning_rate=1.0, line_search=None, line_search_options=None) - + optimizer = Optimizer( model=mlp_model, training_rdd=train_data, @@ -353,7 +357,7 @@ optimizer.setOptimMethod(optimMethod) optim_method = Ftrl(learningrate = 5e-3, \ learningrate_power = -0.5, \ initial_accumulator_value = 0.01) - + optimizer = Optimizer( model=mlp_model, training_rdd=train_data, diff --git a/docs/readthedocs/source/doc/DLlib/Overview/regularizers.md b/docs/readthedocs/source/doc/PythonAPI/DLlib/regularizers.md similarity index 99% rename from docs/readthedocs/source/doc/DLlib/Overview/regularizers.md rename to docs/readthedocs/source/doc/PythonAPI/DLlib/regularizers.md index 162aa164..636f73c8 100644 --- a/docs/readthedocs/source/doc/DLlib/Overview/regularizers.md +++ b/docs/readthedocs/source/doc/PythonAPI/DLlib/regularizers.md @@ -1,3 +1,7 @@ +# Regularizer + +-------- + ## L1 Regularizer ## **Scala:** diff --git a/docs/readthedocs/source/doc/PythonAPI/Friesian/index.rst b/docs/readthedocs/source/doc/PythonAPI/Friesian/index.rst new file mode 100644 index 00000000..43ec676b --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/Friesian/index.rst @@ -0,0 +1,7 @@ +Friesian API +================== + +.. toctree:: + :maxdepth: 2 + + feature.rst \ No newline at end of file diff --git a/docs/readthedocs/source/doc/PythonAPI/Orca/automl.rst b/docs/readthedocs/source/doc/PythonAPI/Orca/automl.rst index 73f06ac0..f5f0cf95 100644 --- a/docs/readthedocs/source/doc/PythonAPI/Orca/automl.rst +++ b/docs/readthedocs/source/doc/PythonAPI/Orca/automl.rst @@ -1,3 +1,6 @@ +Orca AutoML +============================ + orca.automl.auto_estimator --------------------------- @@ -11,7 +14,7 @@ A general estimator supports automatic model tuning. It allows users to fit and orca.automl.hp ---------------------------------------- -Sampling specs to be used in search space configuration. +Sampling specs to be used in search space configuration. .. automodule:: bigdl.orca.automl.hp :members: diff --git a/docs/readthedocs/source/doc/PythonAPI/Orca/context.rst b/docs/readthedocs/source/doc/PythonAPI/Orca/context.rst new file mode 100644 index 00000000..b0bc3687 --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/Orca/context.rst @@ -0,0 +1,15 @@ +Orca Context +========= + +orca.init_orca_context +------------------------- + + +.. automodule:: bigdl.orca.common + :members: init_orca_context + :undoc-members: + :show-inheritance: + + + + diff --git a/docs/readthedocs/source/doc/PythonAPI/Orca/data.rst b/docs/readthedocs/source/doc/PythonAPI/Orca/data.rst new file mode 100644 index 00000000..b4935178 --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/Orca/data.rst @@ -0,0 +1,20 @@ +Orca Data +========= + +orca.data.XShards +--------------------------- + +.. autoclass:: bigdl.orca.data.XShards + :members: + :undoc-members: + :show-inheritance: + + +orca.data.pandas +--------------------------- + +.. automodule:: bigdl.orca.data.pandas.preprocessing + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/readthedocs/source/doc/PythonAPI/Orca/index.rst b/docs/readthedocs/source/doc/PythonAPI/Orca/index.rst new file mode 100644 index 00000000..80a9e1df --- /dev/null +++ b/docs/readthedocs/source/doc/PythonAPI/Orca/index.rst @@ -0,0 +1,10 @@ +Orca API +================== + +.. toctree:: + :maxdepth: 2 + + context.rst + data.rst + orca.rst + automl.rst diff --git a/docs/readthedocs/source/doc/PythonAPI/Orca/orca.rst b/docs/readthedocs/source/doc/PythonAPI/Orca/orca.rst index 5320a5f7..749e94bd 100644 --- a/docs/readthedocs/source/doc/PythonAPI/Orca/orca.rst +++ b/docs/readthedocs/source/doc/PythonAPI/Orca/orca.rst @@ -1,4 +1,4 @@ -Orca API +Orca Learn ========= orca.learn.bigdl.estimator @@ -88,12 +88,3 @@ orca.learn.openvino.estimator :members: :undoc-members: :show-inheritance: - - -AutoML ------------------------------- - -.. toctree:: - :maxdepth: 2 - - automl.rst \ No newline at end of file diff --git a/docs/readthedocs/source/doc/Serving/FAQ/faq.md b/docs/readthedocs/source/doc/Serving/FAQ/faq.md index 350331eb..916ad465 100644 --- a/docs/readthedocs/source/doc/Serving/FAQ/faq.md +++ b/docs/readthedocs/source/doc/Serving/FAQ/faq.md @@ -44,10 +44,10 @@ output of Cluster Serving job information should be displayed, if not, go to [Pr 1. `Duplicate registration of device factory for type XLA_CPU with the same priority 50` -This error is caused by Flink ClassLoader. Please put cluster serving related jars into `${FLINK_HOME}/lib`. + This error is caused by Flink ClassLoader. Please put cluster serving related jars into `${FLINK_HOME}/lib`. 2. `servable Manager config dir not exist` -Check if `servables.yaml` exists in current directory. If not, download from [github](https://github.com/intel-analytics/bigdl/blob/master/ppml/trusted-realtime-ml/scala/docker-graphene/servables.yaml). + Check if `servables.yaml` exists in current directory. If not, download from [github](https://github.com/intel-analytics/bigdl/blob/master/ppml/trusted-realtime-ml/scala/docker-graphene/servables.yaml). ### Still, I get no result If you still get empty result, raise issue [here](https://github.com/intel-analytics/bigdl/issues) and post the output/log of your serving job. diff --git a/docs/readthedocs/source/doc/Serving/index.rst b/docs/readthedocs/source/doc/Serving/index.rst new file mode 100644 index 00000000..de542e40 --- /dev/null +++ b/docs/readthedocs/source/doc/Serving/index.rst @@ -0,0 +1,66 @@ +Cluster Serving +========================= + +BigDL Cluster Serving is a lightweight distributed, real-time serving solution that supports a wide range of deep learning models (such as TensorFlow, PyTorch, Caffe, BigDL and OpenVINO models). It provides a simple pub/sub API, so that the users can easily send their inference requests to the input queue (using a simple Python API); Cluster Serving will then automatically manage the scale-out and real-time model inference across a large cluster (using distributed streaming frameworks such as Apache Spark Streaming, Apache Flink, etc.) + +---------------------- + + + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: + + **Get Started** + ^^^ + + Documents in these sections helps you getting started quickly with Serving. + + +++ + + :bdg-link:`Serving in 5 minutes <./QuickStart/serving-quickstart.html>` | + :bdg-link:`Installation <./ProgrammingGuide/serving-installation.html>` + + .. grid-item-card:: + + **Key Features Guide** + ^^^ + + Each guide in this section provides you with in-depth information, concepts and knowledges about DLLib key features. + + +++ + + :bdg-link:`Start Serving <./ProgrammingGuide/serving-start.html>` | + :bdg-link:`Inference <./ProgrammingGuide/serving-inference.html>` + + + .. grid-item-card:: + + **Examples** + ^^^ + + Cluster Serving Examples and Tutorials. + + +++ + + :bdg-link:`Examples <./Example/example.html>` + + .. grid-item-card:: + + **MISC** + ^^^ + + Cluster Serving + + +++ + + :bdg-link:`FAQ <./FAQ/faq.html>` | + :bdg-link:`Contribute <./FAQ/contribute-guide.html>` + + + +.. toctree:: + :hidden: + + Cluster Serving Document \ No newline at end of file diff --git a/docs/readthedocs/source/doc/UserGuide/develop.md b/docs/readthedocs/source/doc/UserGuide/develop.md index 7ebcf187..a26fcd62 100644 --- a/docs/readthedocs/source/doc/UserGuide/develop.md +++ b/docs/readthedocs/source/doc/UserGuide/develop.md @@ -72,31 +72,31 @@ You need to do the following preparations before starting the IDE to successfull - Build BigDL; see [here](#build) for more instructions. - Prepare Spark environment by either setting `SPARK_HOME` as the environment variable or `pip install pyspark`. Note that the Spark version should match the one you build BigDL on. - Check the jars under `BigDL/dist/lib` and set the environment variable `BIGDL_CLASSPATH`. Modify SPARKVERSION and BIGDLVERSION(Scala) as appropriate: -```bash -export BIGDL_CLASSPATH=BigDL/dist/lib/bigdl-dllib-spark_SPARKVERSION-BIGDLVERSION-jar-with-dependencies.jar:BigDL/dist/lib/bigdl-orca-spark_SPARKVERSION-BIGDLVERSION-jar-with-dependencies.jar:BigDL/dist/lib/bigdl-friesian-spark_SPARKVERSION-BIGDLVERSION-jar-with-dependencies.jar -``` + ```bash + export BIGDL_CLASSPATH=BigDL/dist/lib/bigdl-dllib-spark_SPARKVERSION-BIGDLVERSION-jar-with-dependencies.jar:BigDL/dist/lib/bigdl-orca-spark_SPARKVERSION-BIGDLVERSION-jar-with-dependencies.jar:BigDL/dist/lib/bigdl-friesian-spark_SPARKVERSION-BIGDLVERSION-jar-with-dependencies.jar + ``` - Configure BigDL source files to the Python interpreter: -You can easily do this after launching PyCharm by right clicking the folder `BigDL/python/dllib/src` -> __Mark Directory As__ -> __Sources Root__ (also do this for `BigDL/python/nano/src`, `BigDL/python/orca/src`, `BigDL/python/friesian/src`, `BigDL/python/chronos/src`, `BigDL/python/serving/src` if necessary). + You can easily do this after launching PyCharm by right clicking the folder `BigDL/python/dllib/src` -> __Mark Directory As__ -> __Sources Root__ (also do this for `BigDL/python/nano/src`, `BigDL/python/orca/src`, `BigDL/python/friesian/src`, `BigDL/python/chronos/src`, `BigDL/python/serving/src` if necessary). -Alternatively, you can add BigDL source files to `PYTHONPATH`: -```bash -export PYTHONPATH=BigDL/python/dllib/src:BigDL/python/nano/src:BigDL/python/orca/src:BigDL/python/friesian/src:BigDL/python/chronos/src:BigDL/python/serving/src:$PYTHONPATH -``` + Alternatively, you can add BigDL source files to `PYTHONPATH`: + ```bash + export PYTHONPATH=BigDL/python/dllib/src:BigDL/python/nano/src:BigDL/python/orca/src:BigDL/python/friesian/src:BigDL/python/chronos/src:BigDL/python/serving/src:$PYTHONPATH + ``` - Add `spark-bigdl.conf` to `PYTHONPATH`: -```bash -export PYTHONPATH=BigDL/python/dist/conf/spark-bigdl.conf:$PYTHONPATH -``` + ```bash + export PYTHONPATH=BigDL/python/dist/conf/spark-bigdl.conf:$PYTHONPATH + ``` - Install and add `tflibs` to `TF_LIBS_PATH`: -```bash -# Install bigdl-tf and bigdl-math -pip install bigdl-tf bigdl-math + ```bash + # Install bigdl-tf and bigdl-math + pip install bigdl-tf bigdl-math -# Configure TF_LIBS_PATH -export TF_LIBS_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/bigdl/share/tflibs -``` + # Configure TF_LIBS_PATH + export TF_LIBS_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')/bigdl/share/tflibs + ``` The above environment variables should be available when running or debugging code in the IDE. When running applications in PyCharm, you can add runtime environment variables by clicking __Run__ -> __Edit Configurations__; then in the __Run/Debug Configurations__ panel, you can add necessary environment variables to your applications. diff --git a/docs/readthedocs/source/doc/UserGuide/docker.md b/docs/readthedocs/source/doc/UserGuide/docker.md index cad9bdc6..4d264d78 100644 --- a/docs/readthedocs/source/doc/UserGuide/docker.md +++ b/docs/readthedocs/source/doc/UserGuide/docker.md @@ -125,17 +125,17 @@ After connecting to the Jupyter Notebook in the browser, you can run multiple Bi You should shut down the BigDL Docker container after using it. 1. You can list all the active Docker containers by command line: -``` -sudo docker ps -``` + ``` + sudo docker ps + ``` 2. You will see your docker containers: -``` -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -40de2cdad025 intelanalytics/bigdl:2.1.0-SNAPSHOT "/opt/work/start-n..." 3 hours ago Up 3 hours upbeat_al -``` + ``` + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 40de2cdad025 intelanalytics/bigdl:2.1.0-SNAPSHOT "/opt/work/start-n..." 3 hours ago Up 3 hours upbeat_al + ``` 3. Shut down the corresponding docker container by its ID: -``` -$sudo docker rm -f 40de2cdad025 -``` + ``` + $sudo docker rm -f 40de2cdad025 + ``` diff --git a/docs/readthedocs/source/doc/UserGuide/index.rst b/docs/readthedocs/source/doc/UserGuide/index.rst new file mode 100644 index 00000000..bb8eb2b3 --- /dev/null +++ b/docs/readthedocs/source/doc/UserGuide/index.rst @@ -0,0 +1,60 @@ +.. |scala-logo| image:: ../../../image/scala_logo.png + :height: 20 + :alt: Scala Logo + +User Guide +========================= + +.. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item-card:: :fab:`python` Python User Guide + :link: python.html + :class-card: bigdl-link-card + + Python Environment Setup Guide (Linux), applicable to Orca, Nano, DLlib, Chronos, Friesian. + + .. grid-item-card:: |scala-logo| Scala User Guide + :link: scala.html + :class-card: bigdl-link-card + + Scala Environment Setup Guide (Linux), applicable to DLLib. + + .. grid-item-card:: :fab:`windows` Windows User Guide + :link: win.html + :class-card: bigdl-link-card + + Use BigDL on Windows. + + .. grid-item-card:: :fas:`desktop` Docker User Guide + :link: docker.html + :class-card: bigdl-link-card + + Use BigDL in docker Environment. + + .. grid-item-card:: :fas:`cloud-arrow-up` Colab User Guide + :link: colab.html + :class-card: bigdl-link-card + + Use BigDL in Google Colab Environment. + + .. grid-item-card:: :fas:`cloud` Databricks User Guide + :link: databricks.html + :class-card: bigdl-link-card + + Use BigDL in Databricks Environment. + + .. grid-item-card:: :fas:`cloud` Hadoop/YARN User Guide + :link: hadoop.html + :class-card: bigdl-link-card + + Use BigDL in Hadoop/YARN Environment. + + .. grid-item-card:: :fas:`cloud` k8s User Guide + :link: k8s.html + :class-card: bigdl-link-card + + Use BigDL in K8s Environment. + + + diff --git a/docs/readthedocs/source/doc/UserGuide/k8s.md b/docs/readthedocs/source/doc/UserGuide/k8s.md index 8f6647cf..eb564f00 100644 --- a/docs/readthedocs/source/doc/UserGuide/k8s.md +++ b/docs/readthedocs/source/doc/UserGuide/k8s.md @@ -253,15 +253,15 @@ python script.py ``` In cluster mode, install conda, pack environment and use on both the driver and executor. - Pack the current conda environment to `environment.tar.gz` (you can use any name you like): -```bash -conda pack -o environment.tar.gz -``` + ```bash + conda pack -o environment.tar.gz + ``` - spark-submit with "--archives" and specify python stores for dirver and executor -```bash ---conf spark.pyspark.driver.python=./env/bin/python \ ---conf spark.pyspark.python=./env/bin/python \ ---archives local:///bigdl2.0/data/environment.tar.gz#env \ # this path shoud be that k8s pod can access -``` + ```bash + --conf spark.pyspark.driver.python=./env/bin/python \ + --conf spark.pyspark.python=./env/bin/python \ + --archives local:///bigdl2.0/data/environment.tar.gz#env \ # this path shoud be that k8s pod can access + ``` #### **5.2 How to retain executor logs for debugging?** diff --git a/docs/readthedocs/source/doc/UserGuide/win.md b/docs/readthedocs/source/doc/UserGuide/win.md index b26e5239..edc80ec4 100644 --- a/docs/readthedocs/source/doc/UserGuide/win.md +++ b/docs/readthedocs/source/doc/UserGuide/win.md @@ -86,7 +86,7 @@ jupyter lab ## Tips and Known Issues -### 1.ImportError: libgomp.so.1: cannot open shared object file: No such file or directory +### 1. ImportError: libgomp.so.1: cannot open shared object file: No such file or directory This error may appear when you try to import torch. This is caused by Ubuntu 14.04 or later not installing libgomp1 by default. Just install libgomp1 to resolve it: @@ -94,12 +94,12 @@ This error may appear when you try to import torch. This is caused by Ubuntu 14. sudo apt-get install libgomp1 ``` -### 2.Slow PyTorch training with BF16 +### 2. Slow PyTorch training with BF16 Using BFloat16 mixed precision in PyTorch or PyTorch-Lightning training may be much slower than FP32. -### 3.ERROR: Could not build wheels for pycocotools, which is required to install pyproject.toml-based projects +### 3. ERROR: Could not build wheels for pycocotools, which is required to install pyproject.toml-based projects pycocotools is a dependency of Intel neural-compressor which is used for inference quantization in BigDL-Nano. This error is usually caused by GCC library not installed in system. Just install gcc to resolve it: @@ -107,7 +107,7 @@ pycocotools is a dependency of Intel neural-compressor which is used for inferen sudo apt-get install gcc ``` -### 4.ValueError: After taking into account object store and redis memory usage, the amount of memory on this node available for tasks and actors is less than -75% of total. +### 4. ValueError: After taking into account object store and redis memory usage, the amount of memory on this node available for tasks and actors is less than -75% of total. When running ray applications, you need to set the `memory` and `object_store_memory` properly according to your system memory capacity. This error indicates you have used too large memory configurations and you need to decrease them. For example on a laptop with 8G memory, you may set the memory configurations as below: diff --git a/docs/readthedocs/source/doc/overview.rst b/docs/readthedocs/source/doc/overview.rst new file mode 100644 index 00000000..47651035 --- /dev/null +++ b/docs/readthedocs/source/doc/overview.rst @@ -0,0 +1,2 @@ +Overview +========================= \ No newline at end of file diff --git a/docs/readthedocs/source/index.rst b/docs/readthedocs/source/index.rst index 07c78a30..5a178437 100644 --- a/docs/readthedocs/source/index.rst +++ b/docs/readthedocs/source/index.rst @@ -1,18 +1,71 @@ BigDL Documentation =========================== +BigDL seamlessly scales your data analytics & AI applications from laptop to cloud, with the following libraries: + +- `Orca `_: Distributed Big Data & AI (TF & PyTorch) Pipeline on Spark and Ray +- `Nano `_: Transparent Acceleration of Tensorflow & PyTorch Programs +- `DLlib `_ "Equivalent of Spark MLlib" for Deep Learning +- `Chronos `_: Scalable Time Series Analysis using AutoML +- `Friesian `_: End-to-End Recommendation Systems +- `PPML `_ (experimental): Secure Big Data and AI (with SGX Hardware Security) + ------ -`BigDL `_ makes it easy for data scientists and data engineers to build end-to-end, distributed AI applications. The **BigDL 2.0** release combines the `original BigDL `_ and `Analytics Zoo `_ projects, providing the following features: +Choosing the right BigDL library +--------------------------- -* `DLlib `_: distributed deep learning library for Apache Spark -* `Orca `_: seamlessly scale out TensorFlow and PyTorch pipelines for distributed Big Data -* `RayOnSpark `_: run Ray programs directly on Big Data clusters -* `Chronos `_: scalable time series analysis using AutoML -* `PPML `_: privacy preserving big data analysis and machine learning (*experimental*) -* `Nano `_: automatically accelerate TensorFlow and PyTorch pipelines by applying modern CPU optimizations -------- +.. graphviz:: - -.. meta:: - :google-site-verification: hG9ocvSRSRTY5z8g6RLn97_tdJvYRx_tVGhNdtZZavM + digraph BigDLDecisionTree { + graph [pad=0.1 ranksep=0.3] + node [color="#0171c3" shape=box fontname="Arial" fontsize=14] + + Feature1 [label="Hardware Secured Big Data & AI?"] + Feature2 [label="Python vs. Scala/Java?"] + Feature3 [label="What type of application?"] + Feature4 [label="Domain?"] + + Orca[href="../doc/Orca/index.html" target="_blank" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + Nano[href="../doc/Nano/index.html" target="_blank" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + DLlib1[label="DLlib" href="../doc/DLlib/index.html" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + DLlib2[label="DLlib" href="../doc/DLlib/index.html" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + Chronos[href="../doc/Chronos/index.html" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + Friesian[href="../doc/Friesian/index.html" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + PPML[href="../doc/PPML/index.html" target="_blank" style="rounded,filled" fontcolor="#ffffff"] + + ArrowLabel1[label="No" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel2[label="Yes" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel3[label="Python" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel4[label="Scala/Java" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel5[label="Distributed Big Data \n + \n AI (TF/PyTorch)" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel6[label="Accelerate \n TensorFlow / PyTorch" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel7[label="DL for Spark MLlib" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel8[label="High Level App Framework" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel9[label="Time Series" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + ArrowLabel10[label="Recommender System" fontsize=12 width=0.1 height=0.1 style=filled color="#c9c9c9"] + + Feature1 -> ArrowLabel1[dir=none] + ArrowLabel1 -> Feature2 + Feature1 -> ArrowLabel2[dir=none] + ArrowLabel2 -> PPML + + Feature2 -> ArrowLabel3[dir=none] + ArrowLabel3 -> Feature3 + Feature2 -> ArrowLabel4[dir=none] + ArrowLabel4 -> DLlib1 + + Feature3 -> ArrowLabel5[dir=none] + ArrowLabel5 -> Orca + Feature3 -> ArrowLabel6[dir=none] + ArrowLabel6 -> Nano + Feature3 -> ArrowLabel7[dir=none] + ArrowLabel7 -> DLlib2 + Feature3 -> ArrowLabel8[dir=none] + ArrowLabel8 -> Feature4 + + Feature4 -> ArrowLabel9[dir=none] + ArrowLabel9 -> Chronos + Feature4 -> ArrowLabel10[dir=none] + ArrowLabel10 -> Friesian + } \ No newline at end of file