From a2a9db18bd9cc93fee259b47f7989b606ce45bba Mon Sep 17 00:00:00 2001 From: Orhan Kislal Date: Thu, 2 Mar 2023 16:35:04 +0300 Subject: [PATCH 1/2] Update version numbers to 1.22.0-dev --- RELEASE_NOTES | 2 +- deploy/DEB/postinst | 2 +- deploy/postflight.sh | 2 +- pom.xml | 2 +- src/config/Version.yml | 2 +- .../postgres/modules/deep_learning/madlib_keras.sql_in | 8 ++++---- .../modules/deep_learning/madlib_keras_automl.sql_in | 4 ++-- .../deep_learning/madlib_keras_fit_multiple_model.sql_in | 6 +++--- src/ports/postgres/modules/svm/svm.sql_in | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 50f0f94e8..0734a9637 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -12,7 +12,7 @@ Current list of bugs and issues can be found at https://issues.apache.org/jira/b —------------------------------------------------------------------------- MADlib v1.21.0: -Release Date: 2023-March-XX +Release Date: 2023-March-01 New features: - Graph: Add warm start for weakly connected components diff --git a/deploy/DEB/postinst b/deploy/DEB/postinst index 8569abe55..586adb8da 100755 --- a/deploy/DEB/postinst +++ b/deploy/DEB/postinst @@ -22,7 +22,7 @@ # Source debconf library. . /usr/share/debconf/confmodule -MADLIB_VERSION="1.21.0" +MADLIB_VERSION="1.22.0-dev" MADLIB_INSTALL_PATH="InstallPathNotFound" # Fetching configuration from debconf diff --git a/deploy/postflight.sh b/deploy/postflight.sh index cecd023a9..2750f93cf 100755 --- a/deploy/postflight.sh +++ b/deploy/postflight.sh @@ -2,7 +2,7 @@ # $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volume -MADLIB_VERSION=1.21.0 +MADLIB_VERSION=1.22.0-dev # Remove existing soft links find $2/usr/local/madlib/bin -depth -type l -exec rm {} \; 2>/dev/null diff --git a/pom.xml b/pom.xml index 738d3e548..6d642c0e3 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ org.apache.madlib madlib - 1.21.0 + 1.22.0-dev pom diff --git a/src/config/Version.yml b/src/config/Version.yml index 937913f4f..8df8aefd0 100644 --- a/src/config/Version.yml +++ b/src/config/Version.yml @@ -1 +1 @@ -version: 1.21.0 +version: 1.22.0-dev diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in index a30b394d1..bf3779b5e 100644 --- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in +++ b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in @@ -1143,7 +1143,7 @@ model_size | 0.7900390625 start_training_time | 2021-02-01 15:58:43.760568 end_training_time | 2021-02-01 15:58:44.470054 metrics_elapsed_time | {0.709463119506836} -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {3} dependent_vartype | {"character varying"} normalizing_const | 1 @@ -1457,7 +1457,7 @@ model_size | 0.7900390625 start_training_time | 2021-01-29 14:41:16.943861 end_training_time | 2021-01-29 14:41:19.478149 metrics_elapsed_time | {2.3377411365509,2.42358803749084,2.49885511398315,2.53427410125732} -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {3} dependent_vartype | {"character varying"} normalizing_const | 1 @@ -1627,7 +1627,7 @@ model_size | 0.7900390625 start_training_time | 2021-01-29 14:42:28.780276 end_training_time | 2021-01-29 14:42:31.177561 metrics_elapsed_time | {2.24628114700317,2.28473520278931,2.32178020477295,2.35844302177429,2.39726710319519} -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {3} dependent_vartype | {"character varying"} normalizing_const | 1 @@ -1742,7 +1742,7 @@ model_size | 0.7900390625 start_training_time | 2021-01-29 14:44:51.176983 end_training_time | 2021-01-29 14:44:53.666457 metrics_elapsed_time | {2.48945999145508} -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {3} dependent_vartype | {"character varying"} normalizing_const | 1 diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in index 0447610f5..637da84ca 100644 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in +++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in @@ -1091,7 +1091,7 @@ name | description | start_training_time | 2021-01-16 01:20:17 end_training_time | 2021-01-16 01:21:47 -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | 3 class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | character varying @@ -1220,7 +1220,7 @@ name | description | start_training_time | 2020-10-23 00:24:43 end_training_time | 2020-10-23 00:28:41 -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | 3 class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | character varying diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in index 3664c70a9..33a8c409e 100644 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in +++ b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in @@ -1080,7 +1080,7 @@ name | description | start_training_time | 2021-02-05 00:40:42.695613 end_training_time | 2021-02-05 00:42:20.796712 -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {1} class_text_class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | {"character varying"} @@ -1245,7 +1245,7 @@ name | Sophie L. description | Model selection for iris dataset start_training_time | 2021-02-05 01:03:11.337798 end_training_time | 2021-02-05 01:05:14.988912 -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {1} class_text_class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | {"character varying"} @@ -1419,7 +1419,7 @@ name | Sophie L. description | Simple MLP for iris dataset start_training_time | 2021-02-05 01:17:19.432839 end_training_time | 2021-02-05 01:18:09.062384 -madlib_version | 1.21.0 +madlib_version | 1.22.0-dev num_classes | {1} class_text_class_values | {Iris-setosa,Iris-versicolor,Iris-virginica} dependent_vartype | {"character varying"} diff --git a/src/ports/postgres/modules/svm/svm.sql_in b/src/ports/postgres/modules/svm/svm.sql_in index 131e6be38..3339ebb3d 100644 --- a/src/ports/postgres/modules/svm/svm.sql_in +++ b/src/ports/postgres/modules/svm/svm.sql_in @@ -938,7 +938,7 @@ SELECT * FROM abalone_svm_gaussian_regression_summary; 
 -[ RECORD 1 ]--------+------------------------------------------------------------------------------------
 method               | SVR
-version_number       | 1.21.0
+version_number       | 1.22.0-dev
 source_table         | abalone
 model_table          | abalone_svm_gaussian_regression
 dependent_varname    | rings

From d9a4ee58ad40597ec346c442e80bc8e6c95bffe2 Mon Sep 17 00:00:00 2001
From: Orhan Kislal 
Date: Wed, 19 Apr 2023 17:18:21 -0400
Subject: [PATCH 2/2] WCC: Update docs with warm start example

---
 src/ports/postgres/modules/graph/wcc.sql_in | 36 +++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/ports/postgres/modules/graph/wcc.sql_in b/src/ports/postgres/modules/graph/wcc.sql_in
index 594b74a44..699d348b9 100644
--- a/src/ports/postgres/modules/graph/wcc.sql_in
+++ b/src/ports/postgres/modules/graph/wcc.sql_in
@@ -53,6 +53,8 @@ ignoring the direction of edges. In case of an undirected graph, a weakly
 connected component is also a strongly connected component.  This module also
 includes a number of helper functions that operate on the WCC output.
 
+@note MADlib graph functions might create a large number of subtransactions with very large graphs. Since the cache for subtransaction metadata is limited, running them in conjunction with other processes might degrade the performance of the whole database. The warm start feature of WCC can be used to limit the number of subtransactions in a given session to ensure that the performance is preserved. While the ideal iteration limit is dependent on the graph as well as the rest of the system processes, it is advised to start with 40 and adjust down if the problem persists.
+
 @anchor wcc
 @par Weakly Connected Components
 
@@ -598,6 +600,40 @@ SELECT * FROM wcc_multicol_out ORDER BY user_id_major, user_id_minor, component_
 (14 rows)
 

 
+-# Use iteration limit and warm start in a bash loop to avoid subtx limitations
+
+#!/bin/bash
+psql madlib -c "DROP TABLE IF EXISTS wcc_out, wcc_out_summary, wcc_out_message;"
+\# Run wcc for 2 iterations
+psql madlib -c "SELECT madlib.weakly_connected_components(
+    'vertex',
+    'node_id',
+    'edge',
+    'src=conn_src,dest=conn_dest',
+    'wcc_out',
+    'user_id',
+    2);" # Number of iterations
+\# Check if wcc is done
+mynode=$(psql madlib -tc "SELECT nodes_to_update FROM wcc_out_summary")
+echo $mynode
+\# While there are remaining nodes to update
+while [ $mynode -ne 0 ]
+do
+\# Run WCC with warm start to continue building on the previous output tables
+psql madlib -c "SELECT madlib.weakly_connected_components(
+    'vertex',
+    'node_id',
+    'edge',
+    'src=conn_src,dest=conn_dest',
+    'wcc_out',
+    'user_id',
+    2,
+    True);" # Warm start
+mynode=$(psql madlib -tc "SELECT nodes_to_update FROM wcc_out_summary")
+echo $mynode
+done
+

+
 @anchor notes
 @par Notes