From 060adf485914044efb35b399716d78579404ba2b Mon Sep 17 00:00:00 2001 From: Viacheslav Katsuba Date: Wed, 11 Dec 2019 00:30:05 +0200 Subject: [PATCH 1/4] Mnesia migration Added migration for: pubsub_items migration, users, vcard, vcard_search, event_pusher_push_subscription, rosterusers, roster_versio, last, rostergroups, muc_light_occupants, muc_light_config, migrate_muc_light_rooms, migrate_offline_message, migrate_private_storage --- .../service_admin_extra_migration.erl | 724 ++++++++++++++++++ src/auth/ejabberd_auth_internal.erl | 10 + .../mod_event_pusher_push_mnesia.erl | 17 + src/mod_private_mnesia.erl | 8 + src/muc_light/mod_muc_light_db_mnesia.erl | 17 + src/rdbms/mongoose_rdbms.erl | 3 +- 6 files changed, 778 insertions(+), 1 deletion(-) create mode 100644 src/admin_extra/service_admin_extra_migration.erl diff --git a/src/admin_extra/service_admin_extra_migration.erl b/src/admin_extra/service_admin_extra_migration.erl new file mode 100644 index 00000000000..41372a970f3 --- /dev/null +++ b/src/admin_extra/service_admin_extra_migration.erl @@ -0,0 +1,724 @@ +%%% ================================================================== +%%% @doc +%%% MongooseIM migration +%%% @end +%%% ================================================================== + +-module(service_admin_extra_migration). + +-export([commands/0, migrate/3, get_data/2]). + +-export([ + migrate_pubsub_nodes/1, + migrate_pubsub_affiliations/1, + migrate_pubsub_subscriptions/1, + migrate_pubsub_items/1, + migrate_vcard/1, + migrate_vcard_search/1, + migrate_users/1, + migrate_event_pusher_push_subscription/1, + migrate_rosterusers/1, + migrate_roster_version/1, + migrate_rostergroups/1, + migrate_last/1, + migrate_private_storage/1, + migrate_offline_message/1, + migrate_muc_light_rooms/1 +]). + +%%% ================================================================== +%%% Includes +%%% ================================================================== + +-include("jlib.hrl"). +-include("pubsub.hrl"). +-include("mongoose.hrl"). +-include("mod_last.hrl"). +-include("mod_vcard.hrl"). +-include("mod_roster.hrl"). +-include("mod_offline.hrl"). +-include("ejabberd_commands.hrl"). + +%%% ================================================================== +%%% API +%%% ================================================================== + +%% ------------------------------------------------------------------- +%% @doc +%% Commands of migration CTL +%% @end +%% ------------------------------------------------------------------- +-spec commands() -> [ejabberd_commands:cmd()]. + +commands() -> + [#ejabberd_commands{ + name = migrate, + tags = [migration], + desc = "Migration from Mnesia to MySQL", + module = ?MODULE, + function = migrate, + args = [{from, binary}, {to, binary}, {act, binary}], + result = {res, restuple} + }]. + +%% ------------------------------------------------------------------- +%% @doc +%% Migrate from DB to DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate(From :: binary(), To :: binary(), Act :: binary()) -> Result :: tuple(). + +migrate(<<"mnesia">>, <<"rdbms">>, Act) when is_binary(Act) -> + OP = ejabberd_config:get_local_option_or_default(outgoing_pools, []), + case [X || {X, _, _, _, _} <- OP, X =:= rdbms] of + [rdbms] -> + case try_migrate(Act) of + {ok, Resp} -> + {ok, Resp}; + {error, Reason} -> + {exists, Reason} + end; + [rdbms|_] -> + {exists, "Detected multi 'rdbms' configuration. Need to use one 'rdbms' configuration for correct migration"}; + _ -> + {exists, "Looks like that the MySQL is not configured. Check 'mongooseim.cfg'"} + end; + +migrate(From, To, _) -> + {exists, io_lib:format("Migration from '~s' to '~s' is not supported", [From, To])}. + +%% ------------------------------------------------------------------- +%% @doc +%% @private +%% Trying do migrate from DB to DB +%% @end +%% ------------------------------------------------------------------- +-spec try_migrate(Act :: binary() | tuple()) -> Result :: tuple(). + +try_migrate(Act) when is_binary(Act) -> + case Act of + <<"all">> -> + Acts = [<<"pubsub_nodes">>, <<"pubsub_affiliations">>, <<"pubsub_subscriptions">>, <<"pubsub_items">>, + <<"vcard">>, <<"vcard_search">>, <<"users">>, <<"event_pusher_push_subscription">>, + <<"rosterusers">>, <<"roster_version">>, <<"rostergroups">>, <<"last">>, <<"private_storage">>, + <<"offline_message">>, <<"muc_light_rooms">>], + _ = [try_migrate(X) || X <- Acts], + {ok, io_lib:format("Completed the migration of the tables: ~p", [Acts])}; + <<"pubsub_nodes">> -> + try_migrate({Act, pubsub_node}); + Act when Act =:= <<"pubsub_affiliations">>; Act =:= <<"pubsub_subscriptions">> -> + try_migrate({Act, pubsub_state}); + <<"pubsub_items">> -> + try_migrate({Act, pubsub_item}); + <<"vcard">> -> + try_migrate({Act, vcard}); + <<"vcard_search">> -> + try_migrate({Act, vcard_search}); + <<"users">> -> + try_migrate({Act, passwd}); + <<"event_pusher_push_subscription">> -> + try_migrate({Act, push_subscription}); + Act when Act =:= <<"rosterusers">>; Act =:= <<"rostergroups">> -> + try_migrate({Act, roster}); + <<"roster_version">> -> + try_migrate({Act, roster_version}); + <<"last">> -> + try_migrate({Act, last_activity}); + <<"private_storage">> -> + try_migrate({Act, private_storage}); + <<"offline_message">> -> + try_migrate({Act, offline_msg}); + <<"muc_light_rooms">> -> + try_migrate({Act, muc_light_room}); + Act -> + {error, io_lib:format("Migrate dont support act: ~p", [binary_to_list(Act)])} + end; + +try_migrate({Act, Table}) when is_atom(Table) -> + try_migrate({Act, Table, get_data(first, Table)}); + +try_migrate({Act, Table, TableData}) -> + case TableData of + {ok, _, '$end_of_table'} -> + {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; + {ok, Data, NextKey} -> + ?MODULE:(list_to_existing_atom("migrate_" ++ binary_to_list(Act)))(Data), + try_migrate({Act, Table, get_data(next, {Table, NextKey})}); + Error -> + Error + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "pubsub_node" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_pubsub_nodes(Data :: list()) -> Result :: ok. + +migrate_pubsub_nodes([]) -> + ok; + +migrate_pubsub_nodes([H|_]) -> + {Host, ID} = H#pubsub_node.nodeid, + Owners = jiffy:encode([jid:to_binary(O) || O <- H#pubsub_node.owners]), + Ops = jiffy:encode({H#pubsub_node.options}), + Cols = ["nidx", "p_key", "name", "type", "owners", "options"], + Vals = [H#pubsub_node.id, Host, ID, H#pubsub_node.type, Owners, Ops], + Q = ["INSERT INTO pubsub_nodes ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_node' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "pubsub_affiliations" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_pubsub_affiliations(Data :: list()) -> Result :: ok. + +migrate_pubsub_affiliations([]) -> + ok; + +migrate_pubsub_affiliations([H|_]) -> + {SJID, Nidx} = H#pubsub_state.stateid, + JidBIn = jid:to_binary(SJID), + JID = jid:from_binary(JidBIn), + case {JID#jid.luser, H#pubsub_state.affiliation} of + {Luser, Aff} when Aff =:= none; Luser =:= <<>> -> + ok; + _ -> + JidBIn = jid:to_binary(SJID), + JID = jid:from_binary(JidBIn), + Cols = ["nidx", "luser", "lserver", "aff"], + Vals = [Nidx, JID#jid.luser, JID#jid.lserver, mod_pubsub_db_rdbms:aff2int(H#pubsub_state.affiliation)], + Q = ["INSERT INTO pubsub_affiliations ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_affiliations' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "pubsub_subscriptions" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_pubsub_subscriptions(Data :: list()) -> Result :: ok. + +migrate_pubsub_subscriptions([]) -> + ok; + +migrate_pubsub_subscriptions([H|_]) -> + case H#pubsub_state.subscriptions of + [{Type, SubID}] when Type =:= none; Type =:= pending; Type =:= subscribed -> + {SJID, Nidx} = H#pubsub_state.stateid, + JidBIn = jid:to_binary(SJID), + JID = jid:from_binary(JidBIn), + Cols = ["nidx", "luser", "lserver", "lresource", "type", "sub_id", "options"], + Vals = [Nidx, JID#jid.luser, JID#jid.lserver, JID#jid.lresource, + escape(mod_pubsub_db_rdbms:sub2int(Type)), SubID, "\"{}\""], + Q = ["INSERT INTO pubsub_subscriptions ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_subscriptions' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end; + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "pubsub_items" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_pubsub_items(Data :: list()) -> Result :: ok. + +migrate_pubsub_items([]) -> + ok; + +migrate_pubsub_items([H|_]) -> + {IID, NodeIdx} = H#pubsub_item.itemid, + {CTime, CJID} = H#pubsub_item.creation, + [Payload] = H#pubsub_item.payload, + CT = to_unixtime(CTime), + JidBIn = jid:to_binary(CJID), + JID = jid:from_binary(JidBIn), + XMLB = <<"", (exml:to_binary(Payload))/binary, "">>, + Cols = ["nidx", "itemid", "created_luser", "created_lserver", "created_at", + "modified_luser", "modified_lserver", "modified_lresource", "modified_at", "payload"], + Vals = [NodeIdx, IID, JID#jid.luser, JID#jid.lserver, CT, JID#jid.lserver, + JID#jid.lserver, JID#jid.lresource, CT, XMLB], + Q = ["INSERT INTO pubsub_items ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_items' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "vcard" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_vcard(Data :: list()) -> Result :: ok. + +migrate_vcard([]) -> + ok; + +migrate_vcard([H|_]) -> + {Luser, Lserver} = H#vcard.us, + Cols = ["username", "server", "vcard"], + Vals = [Luser, Lserver, exml:to_binary(H#vcard.vcard)], + Q = ["INSERT INTO vcard ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'vcard' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "vcard_search" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_vcard_search(Data :: list()) -> Result :: ok. + +migrate_vcard_search([]) -> + ok; + +migrate_vcard_search([H|_]) -> + {Luser, Lserver} = H#vcard_search.us, + Cols = ["username", "lusername", "server", "fn", "lfn", "family", "lfamily", "given", + "lgiven", "middle", "lmiddle", "nickname", "lnickname", "bday", "lbday", "ctry", + "lctry", "locality", "llocality", "email", "lemail", "orgname", "lorgname", "orgunit", + "lorgunit"], + Vals = [Luser, H#vcard_search.luser, Lserver, H#vcard_search.fn, H#vcard_search.lfn, + H#vcard_search.family, H#vcard_search.lfamily, H#vcard_search.given, H#vcard_search.lgiven, + H#vcard_search.middle, H#vcard_search.lmiddle, H#vcard_search.nickname, H#vcard_search.lnickname, + H#vcard_search.bday, H#vcard_search.lbday, H#vcard_search.ctry, H#vcard_search.lctry, + H#vcard_search.locality, H#vcard_search.llocality, H#vcard_search.email, H#vcard_search.lemail, + H#vcard_search.orgname, H#vcard_search.lorgname, H#vcard_search.orgunit, H#vcard_search.lorgunit], + Q = ["INSERT INTO vcard_search ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'vcard_search' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "users" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_users(Data :: list()) -> Result :: ok. + +migrate_users([]) -> + ok; + +migrate_users([H|_]) -> + Password = case ejabberd_auth_internal:get_password(H) of + P when is_binary(P) -> + P; + P -> + mongoose_scram:serialize(P) + end, + {Luser, _} = ejabberd_auth_internal:get_us(H), + Cols = ["username", "password"], + Vals = [Luser, Password], + Q = ["INSERT INTO users ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'users' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "event_pusher_push_subscription;" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_event_pusher_push_subscription(Data :: list()) -> Result :: ok. + +migrate_event_pusher_push_subscription([]) -> + ok; + +migrate_event_pusher_push_subscription([H|_]) -> + {Luser, Lserver} = mod_event_pusher_push_mnesia:get_user_jid(H), + Cols = ["owner_jid", "node", "pubsub_jid", "form", "created_at"], + Vals = [<>, mod_event_pusher_push_mnesia:get_pubsub_node(H), + jid:to_binary(mod_event_pusher_push_mnesia:get_pubsub_jid(H)), + jiffy:encode({mod_event_pusher_push_mnesia:get_form(H)}), + integer_to_binary(os:system_time(micro_seconds))], + Q = ["INSERT INTO event_pusher_push_subscription ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'event_pusher_push_subscription' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "rosterusers" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_rosterusers(Data :: list()) -> Result :: ok. + +migrate_rosterusers([]) -> + ok; + +migrate_rosterusers([H|_]) -> + {Luser, _} = H#roster.us, + Cols = ["username", "jid", "nick", "askmessage", "subscription", "ask", "server", "subscribe", "type"], + Vals = [Luser, jid:to_binary(H#roster.jid), H#roster.name, H#roster.askmessage, + subscription_to_char(H#roster.subscription), ask_to_char(H#roster.ask), + "", "", "item"], % @TODO what should be "server" - char(1)??? and what should be "subscribe" and "type" + Q = ["INSERT INTO rosterusers ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'rosterusers' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "roster_version" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_roster_version(Data :: list()) -> Result :: ok. + +migrate_roster_version([]) -> + ok; + +migrate_roster_version([H|_]) -> + {Luser, _} = H#roster_version.us, + Cols = ["username", "version"], + Vals = [Luser, H#roster_version.version], + Q = ["INSERT INTO roster_version ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'roster_version' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "rostergroups" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_rostergroups(Data :: list()) -> Result :: ok. + +migrate_rostergroups([]) -> + ok; + +migrate_rostergroups([H|_]) -> + {Luser, _} = H#roster.us, + Cols = ["username", "jid", "grp"], + Vals = [Luser, jid:to_binary(H#roster.jid)], + _ = [begin + Q = ["INSERT INTO rostergroups ", expand_sql_vals(Cols, Vals ++ [Group]), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'rostergroups' has error: ~p for values ~p", [Reason, Vals ++ Group]); + _ -> + ok + end + end || Group <- H#roster.groups], + ok. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "last" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_last(Data :: list()) -> Result :: ok. + +migrate_last([]) -> + ok; + +migrate_last([H|_]) -> + {Luser, _} = H#last_activity.us, + Cols = ["username", "seconds", "state"], + Vals = [Luser, H#last_activity.timestamp, H#last_activity.status], + Q = ["INSERT INTO last ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'last' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "private_storage" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_private_storage(Data :: list()) -> Result :: ok. + +migrate_private_storage([]) -> + ok; + +migrate_private_storage([H|_]) -> + {LUser, _, NS} = mod_private_mnesia:get_usns(H), + Cols = ["username", "namespace", "data"], + Vals = [LUser, NS, exml:to_binary(mod_private_mnesia:get_xml(H))], + Q = ["INSERT INTO private_storage ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'private_storage' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "offline_message" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_offline_message(Data :: list()) -> Result :: ok. + +migrate_offline_message([]) -> + ok; + +migrate_offline_message([H|_]) -> + {LUser, LServer} = H#offline_msg.us, + Cols = ["timestamp", "server", "username", "from_jid", "packet"], + Vals = [integer_to_binary(to_unixtime(H#offline_msg.timestamp)), LServer, + LUser, jid:to_binary(H#offline_msg.from), exml:to_binary(H#offline_msg.packet)], + Q = ["INSERT INTO offline_message ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'offline_message' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "muc_light_rooms" from Mnesia to SQL DB +%% @end +%% ------------------------------------------------------------------- +-spec migrate_muc_light_rooms(Data :: list()) -> Result :: ok. + +migrate_muc_light_rooms([]) -> + ok; + +migrate_muc_light_rooms([H|_]) -> + {LUser, LServer} = mod_muc_light_db_mnesia:get_room(H), + ID = case mongoose_rdbms:sql_query(?MYNAME, "SELECT MAX(ID) from muc_light_rooms;") of + {selected, [{null}]} -> + 1; + {selected, [{N}]} when is_integer(N) -> + N + 1 + end, + Cols = ["id", "luser", "lserver", "version"], + Vals = [ID, LUser, LServer, mod_muc_light_db_mnesia:get_room_version(H)], + Q = ["INSERT INTO muc_light_rooms ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'muc_light_rooms' has error: ~p for values ~p", [Reason, Vals]); + _ -> + migrate_muc_light_config(mod_muc_light_db_mnesia:get_room_config(H), ID), + _ = [migrate_muc_light_occupants(ID, U, S, Aff) || {{U, S}, Aff} <- mod_muc_light_db_mnesia:get_room_aff_users(H)], + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "muc_light_config" from Mnesia to SQL DB +%% For "muc_light_config" need to know "room_id", by this reason this function used inside of migrate_muc_light_rooms/1 +%% @end +%% ------------------------------------------------------------------- +-spec migrate_muc_light_config(Data :: list(), RoomID :: integer()) -> ok. + +migrate_muc_light_config([{_, Name}, {_, Subject}], RoomID) -> + Cols = ["room_id", "opt", "val"], + Vals = [RoomID, Name, Subject], + Q = ["INSERT INTO muc_light_config ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'muc_light_config' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Helper for migrate "muc_light_occupants" from Mnesia to SQL DB +%% For "muc_light_occupants" need to know "room_id", by this reason this function used inside of migrate_muc_light_rooms/1 +%% @end +%% ------------------------------------------------------------------- +-spec migrate_muc_light_occupants(ID :: integer(), U :: binary(), S :: binary(), A :: atom()) -> ok. + +migrate_muc_light_occupants(ID, LUser, LServer, Aff) -> + AffDB = case Aff of + owner -> + 1; + member -> + 2; + _ -> + 0 + end, + case AffDB of + 0 -> + ok; + AffDB -> + Cols = ["room_id", "luser", "lserver", "aff"], + Vals = [ID, LUser, LServer, AffDB], + Q = ["INSERT INTO muc_light_occupants ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'migrate_muc_light_occupants' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% convert time for DB +%% @end +%% ------------------------------------------------------------------- +-spec to_unixtime(Data :: tuple()) -> Result :: integer(). + +to_unixtime({Mega, Sec, Micro}) -> + Mega * 1000000 * 1000000 + Sec * 1000000 + Micro. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Getting data from table of Mnesia by table name +%% @end +%% ------------------------------------------------------------------- +-spec get_data(Step :: atom(), Name :: atom() | tuple()) -> Result :: {ok, any(), any()} | {error, list()}. + +get_data(first, Name) -> + try + Key = mnesia:dirty_first(Name), + {ok, mnesia:dirty_read(Name, Key), mnesia:dirty_next(Name, Key)} + catch Class:Exception -> + {error, io_lib:format("Migration call: [~p:~p/1] catched ~p:~p", [?MODULE, get_data, Class, Exception])} + end; + +get_data(next, {Name, Key}) -> + {ok, mnesia:dirty_read(Name, Key), mnesia:dirty_next(Name, Key)}. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Generate SQL values: "(field_1, field_1) VALUES ('bear', 'one')" +%% @end +%% ------------------------------------------------------------------- +-spec expand_sql_vals(Cols :: list(), Vals :: list()) -> Q :: list(). + +expand_sql_vals(Cols, Vals) -> + EVals = [escape(V) || V <- Vals], + ["(", join(Cols, ", "), ") VALUES (", join(EVals, ", "), ")"]. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Join lists by separator +%% @end +%% ------------------------------------------------------------------- +-spec join(list(), list()) -> list(). + +join([], _Sep) -> + []; + +join([H|T], Sep) -> + [H, [[Sep, X] || X <- T]]. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% escape helper +%% @end +%% ------------------------------------------------------------------- +-spec escape(Data :: any()) -> Res :: binary() | list() | any(). + +escape(Data) when is_integer(Data) -> + integer_to_binary(Data); + +escape(Data) when is_binary(Data); is_list(Data) -> + [$', mongoose_rdbms:escape_characters(Data), $']; + +escape(Data) -> + Data. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Subscription to char +%% @end +%% ------------------------------------------------------------------- +-spec subscription_to_char(Data :: atom()) -> Result :: binary(). + +subscription_to_char(Subs) -> + case Subs of + both -> + <<"B">>; + to -> + <<"T">>; + from -> + <<"F">>; + remove -> + <<"R">>; + none -> + <<"N">> + end. + +%% ------------------------------------------------------------------- +%% @private +%% @doc +%% Ask to char +%% @end +%% ------------------------------------------------------------------- +-spec ask_to_char(Data :: atom()) -> Result :: binary(). + +ask_to_char(Ask) -> + case Ask of + subscribe -> + <<"S">>; + unsubscribe -> + <<"U">>; + both -> + <<"B">>; + out -> + <<"O">>; + in -> + <<"I">>; + _ -> + <<"N">> + end. diff --git a/src/auth/ejabberd_auth_internal.erl b/src/auth/ejabberd_auth_internal.erl index 18c3afb1dbe..22ef11b93be 100644 --- a/src/auth/ejabberd_auth_internal.erl +++ b/src/auth/ejabberd_auth_internal.erl @@ -51,6 +51,8 @@ -export([check_password/3, check_password/5]). +-export([get_us/1, get_password/1]). + -include("mongoose.hrl"). -include("scram.hrl"). @@ -361,3 +363,11 @@ get_vh_registered_users_within_interval(Users, Limit, Offset) -> Length = length(Set), Start = min(1, max(Offset, Length)), lists:sublist(Set, Start, Limit). + +-spec get_us(Data :: passwd()) -> tuple(). +get_us(#passwd{us = US}) -> + US. + +-spec get_password(Data :: passwd()) -> tuple() | binary(). +get_password(#passwd{password = Pass}) -> + Pass. diff --git a/src/event_pusher/mod_event_pusher_push_mnesia.erl b/src/event_pusher/mod_event_pusher_push_mnesia.erl index 6d270972b35..d5146262f15 100644 --- a/src/event_pusher/mod_event_pusher_push_mnesia.erl +++ b/src/event_pusher/mod_event_pusher_push_mnesia.erl @@ -19,6 +19,7 @@ -export([init/2]). -export([enable/4, disable/3, get_publish_services/1]). +-export([get_user_jid/1, get_pubsub_jid/1, get_pubsub_node/1, get_form/1]). %%-------------------------------------------------------------------- %% Definitions @@ -148,3 +149,19 @@ make_record(UserJID, PubsubJID, Node, Form) -> -spec key(jid:jid()) -> key(). key(JID) -> jid:to_lus(JID). + +-spec get_user_jid(Data :: tuple()) -> Result :: key() | undefined. +get_user_jid(#push_subscription{user_jid = UserJid}) -> + UserJid. + +-spec get_pubsub_jid(Data :: tuple()) -> Result :: jid:jid(). +get_pubsub_jid(#push_subscription{pubsub_jid = PubsubJid}) -> + PubsubJid. + +-spec get_pubsub_node(Data :: tuple()) -> Result :: mod_event_pusher_push:pubsub_node(). +get_pubsub_node(#push_subscription{pubsub_node = PubsubNode}) -> + PubsubNode. + +-spec get_form(Data :: tuple()) -> Result :: mod_event_pusher_push:form(). +get_form(#push_subscription{form = Form}) -> + Form. diff --git a/src/mod_private_mnesia.erl b/src/mod_private_mnesia.erl index ba76c990114..a26bc2a4ac0 100644 --- a/src/mod_private_mnesia.erl +++ b/src/mod_private_mnesia.erl @@ -37,6 +37,8 @@ -export([get_all_nss/2]). +-export([get_usns/1, get_xml/1]). + -include("mongoose.hrl"). -include("jlib.hrl"). @@ -98,3 +100,9 @@ select_namespaces_t(LUser, LServer) -> delete_record_t(LUser, LServer, NS) -> mnesia:delete({private_storage, {LUser, LServer, NS}}). + +get_usns(#private_storage{usns = USNS}) -> + USNS. + +get_xml(#private_storage{xml = XML}) -> + XML. diff --git a/src/muc_light/mod_muc_light_db_mnesia.erl b/src/muc_light/mod_muc_light_db_mnesia.erl index 09eb86122cd..17bf0223b1c 100644 --- a/src/muc_light/mod_muc_light_db_mnesia.erl +++ b/src/muc_light/mod_muc_light_db_mnesia.erl @@ -55,6 +55,8 @@ force_clear/0 ]). +-export([get_room/1, get_room_config/1, get_room_aff_users/1, get_room_version/1]). + -include("mongoose.hrl"). -include("jlib.hrl"). -include("mod_muc_light.hrl"). @@ -414,3 +416,18 @@ update_users_rooms(RoomUS, [], [User | RLeavingUsers]) -> update_users_rooms(_RoomUS, [], []) -> ok. +-spec get_room(Data :: muc_light_room()) -> jid:simple_bare_jid(). +get_room(#muc_light_room{room = Room}) -> + Room. + +-spec get_room_config(Data :: muc_light_room()) -> [{atom(), term()}]. +get_room_config(#muc_light_room{config = Config}) -> + Config. + +-spec get_room_aff_users(Data :: muc_light_room()) -> aff_users(). +get_room_aff_users(#muc_light_room{aff_users = AffUsers}) -> + AffUsers. + +-spec get_room_version(Data :: muc_light_room()) -> binary(). +get_room_version(#muc_light_room{version = Version}) -> + Version. diff --git a/src/rdbms/mongoose_rdbms.erl b/src/rdbms/mongoose_rdbms.erl index f92c3ca669d..082a33dbb11 100644 --- a/src/rdbms/mongoose_rdbms.erl +++ b/src/rdbms/mongoose_rdbms.erl @@ -89,7 +89,8 @@ %% Unicode escaping -export([escape_string/1, - use_escaped_string/1]). + use_escaped_string/1, + escape_characters/1]). %% Integer escaping -export([escape_integer/1, From 66a02b495f5129b9207e1dfff5e53ff438f840a4 Mon Sep 17 00:00:00 2001 From: vkatsuba Date: Thu, 17 Dec 2020 13:54:01 +0200 Subject: [PATCH 2/4] Mnesia migration: CT --- big_tests/default.spec | 1 + big_tests/tests/mnesia_migration_SUITE.erl | 210 +++++ rel/files/mongooseim.toml | 2 +- .../service_admin_extra_migration.erl | 780 +++++++++--------- 4 files changed, 611 insertions(+), 382 deletions(-) create mode 100644 big_tests/tests/mnesia_migration_SUITE.erl diff --git a/big_tests/default.spec b/big_tests/default.spec index dd99a73a34d..2673221381a 100644 --- a/big_tests/default.spec +++ b/big_tests/default.spec @@ -80,6 +80,7 @@ {suites, "tests", vcard_simple_SUITE}. {suites, "tests", websockets_SUITE}. {suites, "tests", xep_0352_csi_SUITE}. +{suites, "tests", mnesia_migration_SUITE}. {config, ["test.config"]}. {logdir, "ct_report"}. diff --git a/big_tests/tests/mnesia_migration_SUITE.erl b/big_tests/tests/mnesia_migration_SUITE.erl new file mode 100644 index 00000000000..b5c8a0b41c4 --- /dev/null +++ b/big_tests/tests/mnesia_migration_SUITE.erl @@ -0,0 +1,210 @@ +-module(mnesia_migration_SUITE). + +-compile(export_all). + +%%% ================================================================== +%%% Includes +%%% ================================================================== + +-include_lib("escalus/include/escalus.hrl"). +-include_lib("exml/include/exml.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%%% ================================================================== +%%% Macro +%%% ================================================================== + +-define(RPC_MIGRATE(Act), mongoose_helper:successful_rpc(service_admin_extra_migration, migrate, [<<"mnesia">>, <<"rdbms">>, Act])). + +%%-------------------------------------------------------------------- +%% Suite configuration +%%-------------------------------------------------------------------- + +all() -> + case (not ct_helper:is_ct_running()) orelse mongoose_helper:is_rdbms_enabled(ct:get_config({hosts, mim, domain})) of + true -> + tests(); + false -> + {skip, require_rdbms} + end. + +tests() -> + [ + {group, migration} + ]. + +groups() -> + G = + [ + {migration, [sequence], + [ + migrate_pubsub_nodes, + migrate_pubsub_subscriptions, + migrate_pubsub_affiliations, + migrate_pubsub_items, + migrate_users, + migrate_vcard_search, + migrate_vcard, + migrate_event_pusher_push_subscription, + migrate_rosterusers, + migrate_roster_version, + migrate_rostergroups, + migrate_last, + migrate_private_storage, + migrate_offline_message, + migrate_muc_light_rooms, + migrate_all + ] + } + ], + ct_helper:repeat_all_until_all_ok(G). + +suite() -> + escalus:suite(). + +init_per_suite(Config) -> + escalus:init_per_suite(Config). + +end_per_suite(Config) -> + escalus_fresh:clean(), + escalus:end_per_suite(Config). + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +init_per_testcase(CaseName, Config) -> + escalus:init_per_testcase(CaseName, Config). + +end_per_testcase(CaseName, Config) -> + escalus:end_per_testcase(CaseName, Config). + +%%% ================================================================== +%%% Test case for migrate pubsub_nodes @TODO +%%% ================================================================== + +migrate_pubsub_nodes(Config) -> + R = ?RPC_MIGRATE(<<"pubsub_nodes">>), + ct:comment("TEST CASE ~p", [{?FUNCTION_NAME, R}]). + +%%% ================================================================== +%%% Test case for migrate pubsub_subscriptions @TODO +%%% ================================================================== + +migrate_pubsub_subscriptions(Config) -> + ?RPC_MIGRATE(<<"pubsub_subscriptions">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate pubsub_affiliations @TODO +%%% ================================================================== + +migrate_pubsub_affiliations(Config) -> + ?RPC_MIGRATE(<<"pubsub_affiliations">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate pubsub_items @TODO +%%% ================================================================== + +migrate_pubsub_items(Config) -> + ?RPC_MIGRATE(<<"pubsub_items">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate users @TODO +%%% ================================================================== + +migrate_users(Config) -> + ?RPC_MIGRATE(<<"users">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate vcard_search @TODO +%%% ================================================================== + +migrate_vcard_search(Config) -> + ?RPC_MIGRATE(<<"vcard_search">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate vcard @TODO +%%% ================================================================== + +migrate_vcard(Config) -> + ?RPC_MIGRATE(<<"vcard">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate event_pusher_push_subscription @TODO +%%% ================================================================== + +migrate_event_pusher_push_subscription(Config) -> + ?RPC_MIGRATE(<<"event_pusher_push_subscription">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate rosterusers @TODO +%%% ================================================================== + +migrate_rosterusers(Config) -> + ?RPC_MIGRATE(<<"rosterusers">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate roster_version @TODO +%%% ================================================================== + +migrate_roster_version(Config) -> + ?RPC_MIGRATE(<<"roster_version">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate rostergroups @TODO +%%% ================================================================== + +migrate_rostergroups(Config) -> + ?RPC_MIGRATE(<<"rostergroups">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate last @TODO +%%% ================================================================== + +migrate_last(Config) -> + ?RPC_MIGRATE(<<"last">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate private_storage @TODO +%%% ================================================================== + +migrate_private_storage(Config) -> + ?RPC_MIGRATE(<<"private_storage">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate offline_message @TODO +%%% ================================================================== + +migrate_offline_message(Config) -> + ?RPC_MIGRATE(<<"offline_message">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate muc_light_rooms @TODO +%%% ================================================================== + +migrate_muc_light_rooms(Config) -> + ?RPC_MIGRATE(<<"muc_light_rooms">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Test case for migrate all @TODO +%%% ================================================================== + +migrate_all(Config) -> + ?RPC_MIGRATE(<<"all">>), + ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). diff --git a/rel/files/mongooseim.toml b/rel/files/mongooseim.toml index ba105c5c266..cefb86eea6a 100644 --- a/rel/files/mongooseim.toml +++ b/rel/files/mongooseim.toml @@ -199,7 +199,7 @@ [services.service_admin_extra] submods = ["node", "accounts", "sessions", "vcard", "gdpr", "upload", - "roster", "last", "private", "stanza", "stats"] + "roster", "last", "private", "stanza", "stats", "migration"] [services.service_mongoose_system_metrics] initial_report = 300_000 diff --git a/src/admin_extra/service_admin_extra_migration.erl b/src/admin_extra/service_admin_extra_migration.erl index 41372a970f3..726afc101ca 100644 --- a/src/admin_extra/service_admin_extra_migration.erl +++ b/src/admin_extra/service_admin_extra_migration.erl @@ -1,29 +1,47 @@ -%%% ================================================================== -%%% @doc -%%% MongooseIM migration -%%% @end -%%% ================================================================== +%%%---------------------------------------------------------------------- +%%% File : service_admin_extra_migration.erl +%%% Author : Viacheslav Katsuba +%%% Purpose : Migration Mnesia to RDBMS +%%% Created : 11 Dec 2019 by Viacheslav Katsuba +%%% +%%% MongooseIM, Copyright (C) 2020 Erlang Solutions Ltd. +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License +%%% along with this program; if not, write to the Free Software +%%% Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +%%% +%%%---------------------------------------------------------------------- -module(service_admin_extra_migration). -export([commands/0, migrate/3, get_data/2]). -export([ - migrate_pubsub_nodes/1, - migrate_pubsub_affiliations/1, - migrate_pubsub_subscriptions/1, - migrate_pubsub_items/1, - migrate_vcard/1, - migrate_vcard_search/1, - migrate_users/1, - migrate_event_pusher_push_subscription/1, - migrate_rosterusers/1, - migrate_roster_version/1, - migrate_rostergroups/1, - migrate_last/1, - migrate_private_storage/1, - migrate_offline_message/1, - migrate_muc_light_rooms/1 + migrate_pubsub_nodes/1, + migrate_pubsub_affiliations/1, + migrate_pubsub_subscriptions/1, + migrate_pubsub_items/1, + migrate_vcard/1, + migrate_vcard_search/1, + migrate_users/1, + migrate_event_pusher_push_subscription/1, + migrate_rosterusers/1, + migrate_roster_version/1, + migrate_rostergroups/1, + migrate_last/1, + migrate_private_storage/1, + migrate_offline_message/1, + migrate_muc_light_rooms/1 ]). %%% ================================================================== @@ -51,14 +69,14 @@ -spec commands() -> [ejabberd_commands:cmd()]. commands() -> - [#ejabberd_commands{ - name = migrate, - tags = [migration], - desc = "Migration from Mnesia to MySQL", - module = ?MODULE, - function = migrate, - args = [{from, binary}, {to, binary}, {act, binary}], - result = {res, restuple} + [#ejabberd_commands{ + name = migrate, + tags = [migration], + desc = "Migration from Mnesia to MySQL", + module = ?MODULE, + function = migrate, + args = [{from, binary}, {to, binary}, {act, binary}], + result = {res, restuple} }]. %% ------------------------------------------------------------------- @@ -69,23 +87,23 @@ commands() -> -spec migrate(From :: binary(), To :: binary(), Act :: binary()) -> Result :: tuple(). migrate(<<"mnesia">>, <<"rdbms">>, Act) when is_binary(Act) -> - OP = ejabberd_config:get_local_option_or_default(outgoing_pools, []), - case [X || {X, _, _, _, _} <- OP, X =:= rdbms] of - [rdbms] -> - case try_migrate(Act) of - {ok, Resp} -> - {ok, Resp}; - {error, Reason} -> - {exists, Reason} - end; - [rdbms|_] -> - {exists, "Detected multi 'rdbms' configuration. Need to use one 'rdbms' configuration for correct migration"}; - _ -> - {exists, "Looks like that the MySQL is not configured. Check 'mongooseim.cfg'"} + OP = ejabberd_config:get_local_option_or_default(outgoing_pools, []), + case [X || {X, _, _, _, _} <- OP, X =:= rdbms] of + [rdbms] -> + case try_migrate(Act) of + {ok, Resp} -> + {ok, Resp}; + {error, Reason} -> + {exists, Reason} + end; + [rdbms|_] -> + {exists, "Detected multi 'rdbms' configuration. Need to use one 'rdbms' configuration for correct migration"}; + _ -> + {exists, "Looks like that the MySQL is not configured. Check 'mongooseim.cfg'"} end; migrate(From, To, _) -> - {exists, io_lib:format("Migration from '~s' to '~s' is not supported", [From, To])}. + {exists, io_lib:format("Migration from '~s' to '~s' is not supported", [From, To])}. %% ------------------------------------------------------------------- %% @doc @@ -96,57 +114,57 @@ migrate(From, To, _) -> -spec try_migrate(Act :: binary() | tuple()) -> Result :: tuple(). try_migrate(Act) when is_binary(Act) -> - case Act of - <<"all">> -> - Acts = [<<"pubsub_nodes">>, <<"pubsub_affiliations">>, <<"pubsub_subscriptions">>, <<"pubsub_items">>, - <<"vcard">>, <<"vcard_search">>, <<"users">>, <<"event_pusher_push_subscription">>, - <<"rosterusers">>, <<"roster_version">>, <<"rostergroups">>, <<"last">>, <<"private_storage">>, - <<"offline_message">>, <<"muc_light_rooms">>], - _ = [try_migrate(X) || X <- Acts], - {ok, io_lib:format("Completed the migration of the tables: ~p", [Acts])}; - <<"pubsub_nodes">> -> - try_migrate({Act, pubsub_node}); - Act when Act =:= <<"pubsub_affiliations">>; Act =:= <<"pubsub_subscriptions">> -> - try_migrate({Act, pubsub_state}); - <<"pubsub_items">> -> - try_migrate({Act, pubsub_item}); - <<"vcard">> -> - try_migrate({Act, vcard}); - <<"vcard_search">> -> - try_migrate({Act, vcard_search}); - <<"users">> -> - try_migrate({Act, passwd}); - <<"event_pusher_push_subscription">> -> - try_migrate({Act, push_subscription}); - Act when Act =:= <<"rosterusers">>; Act =:= <<"rostergroups">> -> - try_migrate({Act, roster}); - <<"roster_version">> -> - try_migrate({Act, roster_version}); - <<"last">> -> - try_migrate({Act, last_activity}); - <<"private_storage">> -> - try_migrate({Act, private_storage}); - <<"offline_message">> -> - try_migrate({Act, offline_msg}); - <<"muc_light_rooms">> -> - try_migrate({Act, muc_light_room}); - Act -> - {error, io_lib:format("Migrate dont support act: ~p", [binary_to_list(Act)])} - end; + case Act of + <<"all">> -> + Acts = [<<"pubsub_nodes">>, <<"pubsub_affiliations">>, <<"pubsub_subscriptions">>, <<"pubsub_items">>, + <<"vcard">>, <<"vcard_search">>, <<"users">>, <<"event_pusher_push_subscription">>, + <<"rosterusers">>, <<"roster_version">>, <<"rostergroups">>, <<"last">>, <<"private_storage">>, + <<"offline_message">>, <<"muc_light_rooms">>], + _ = [try_migrate(X) || X <- Acts], + {ok, io_lib:format("Completed the migration of the tables: ~p", [Acts])}; + <<"pubsub_nodes">> -> + try_migrate({Act, pubsub_node}); + Act when Act =:= <<"pubsub_affiliations">>; Act =:= <<"pubsub_subscriptions">> -> + try_migrate({Act, pubsub_state}); + <<"pubsub_items">> -> + try_migrate({Act, pubsub_item}); + <<"vcard">> -> + try_migrate({Act, vcard}); + <<"vcard_search">> -> + try_migrate({Act, vcard_search}); + <<"users">> -> + try_migrate({Act, passwd}); + <<"event_pusher_push_subscription">> -> + try_migrate({Act, push_subscription}); + Act when Act =:= <<"rosterusers">>; Act =:= <<"rostergroups">> -> + try_migrate({Act, roster}); + <<"roster_version">> -> + try_migrate({Act, roster_version}); + <<"last">> -> + try_migrate({Act, last_activity}); + <<"private_storage">> -> + try_migrate({Act, private_storage}); + <<"offline_message">> -> + try_migrate({Act, offline_msg}); + <<"muc_light_rooms">> -> + try_migrate({Act, muc_light_room}); + Act -> + {error, io_lib:format("Migrate dont support act: ~p", [binary_to_list(Act)])} + end; try_migrate({Act, Table}) when is_atom(Table) -> - try_migrate({Act, Table, get_data(first, Table)}); + try_migrate({Act, Table, get_data(first, Table)}); try_migrate({Act, Table, TableData}) -> - case TableData of - {ok, _, '$end_of_table'} -> - {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; - {ok, Data, NextKey} -> - ?MODULE:(list_to_existing_atom("migrate_" ++ binary_to_list(Act)))(Data), - try_migrate({Act, Table, get_data(next, {Table, NextKey})}); - Error -> - Error - end. + case TableData of + {ok, _, '$end_of_table'} -> + {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; + {ok, Data, NextKey} -> + ?MODULE:(list_to_existing_atom("migrate_" ++ binary_to_list(Act)))(Data), + try_migrate({Act, Table, get_data(next, {Table, NextKey})}); + Error -> + Error + end. %% ------------------------------------------------------------------- %% @private @@ -157,21 +175,21 @@ try_migrate({Act, Table, TableData}) -> -spec migrate_pubsub_nodes(Data :: list()) -> Result :: ok. migrate_pubsub_nodes([]) -> - ok; + ok; migrate_pubsub_nodes([H|_]) -> - {Host, ID} = H#pubsub_node.nodeid, - Owners = jiffy:encode([jid:to_binary(O) || O <- H#pubsub_node.owners]), - Ops = jiffy:encode({H#pubsub_node.options}), - Cols = ["nidx", "p_key", "name", "type", "owners", "options"], - Vals = [H#pubsub_node.id, Host, ID, H#pubsub_node.type, Owners, Ops], - Q = ["INSERT INTO pubsub_nodes ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'pubsub_node' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Host, ID} = H#pubsub_node.nodeid, + Owners = jiffy:encode([jid:to_binary(O) || O <- H#pubsub_node.owners]), + Ops = jiffy:encode({H#pubsub_node.options}), + Cols = ["nidx", "p_key", "name", "type", "owners", "options"], + Vals = [H#pubsub_node.id, Host, ID, H#pubsub_node.type, Owners, Ops], + Q = ["INSERT INTO pubsub_nodes ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_node' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -182,28 +200,28 @@ migrate_pubsub_nodes([H|_]) -> -spec migrate_pubsub_affiliations(Data :: list()) -> Result :: ok. migrate_pubsub_affiliations([]) -> - ok; + ok; migrate_pubsub_affiliations([H|_]) -> - {SJID, Nidx} = H#pubsub_state.stateid, - JidBIn = jid:to_binary(SJID), - JID = jid:from_binary(JidBIn), - case {JID#jid.luser, H#pubsub_state.affiliation} of - {Luser, Aff} when Aff =:= none; Luser =:= <<>> -> - ok; - _ -> - JidBIn = jid:to_binary(SJID), - JID = jid:from_binary(JidBIn), - Cols = ["nidx", "luser", "lserver", "aff"], - Vals = [Nidx, JID#jid.luser, JID#jid.lserver, mod_pubsub_db_rdbms:aff2int(H#pubsub_state.affiliation)], - Q = ["INSERT INTO pubsub_affiliations ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'pubsub_affiliations' has error: ~p for values ~p", [Reason, Vals]); + {SJID, Nidx} = H#pubsub_state.stateid, + JidBIn = jid:to_binary(SJID), + JID = jid:from_binary(JidBIn), + case {JID#jid.luser, H#pubsub_state.affiliation} of + {Luser, Aff} when Aff =:= none; Luser =:= <<>> -> + ok; _ -> - ok - end - end. + JidBIn = jid:to_binary(SJID), + JID = jid:from_binary(JidBIn), + Cols = ["nidx", "luser", "lserver", "aff"], + Vals = [Nidx, JID#jid.luser, JID#jid.lserver, mod_pubsub_db_rdbms:aff2int(H#pubsub_state.affiliation)], + Q = ["INSERT INTO pubsub_affiliations ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_affiliations' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end + end. %% ------------------------------------------------------------------- %% @private @@ -214,27 +232,27 @@ migrate_pubsub_affiliations([H|_]) -> -spec migrate_pubsub_subscriptions(Data :: list()) -> Result :: ok. migrate_pubsub_subscriptions([]) -> - ok; + ok; migrate_pubsub_subscriptions([H|_]) -> - case H#pubsub_state.subscriptions of - [{Type, SubID}] when Type =:= none; Type =:= pending; Type =:= subscribed -> - {SJID, Nidx} = H#pubsub_state.stateid, - JidBIn = jid:to_binary(SJID), - JID = jid:from_binary(JidBIn), - Cols = ["nidx", "luser", "lserver", "lresource", "type", "sub_id", "options"], - Vals = [Nidx, JID#jid.luser, JID#jid.lserver, JID#jid.lresource, - escape(mod_pubsub_db_rdbms:sub2int(Type)), SubID, "\"{}\""], - Q = ["INSERT INTO pubsub_subscriptions ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'pubsub_subscriptions' has error: ~p for values ~p", [Reason, Vals]); + case H#pubsub_state.subscriptions of + [{Type, SubID}] when Type =:= none; Type =:= pending; Type =:= subscribed -> + {SJID, Nidx} = H#pubsub_state.stateid, + JidBIn = jid:to_binary(SJID), + JID = jid:from_binary(JidBIn), + Cols = ["nidx", "luser", "lserver", "lresource", "type", "sub_id", "options"], + Vals = [Nidx, JID#jid.luser, JID#jid.lserver, JID#jid.lresource, + escape(mod_pubsub_db_rdbms:sub2int(Type)), SubID, "\"{}\""], + Q = ["INSERT INTO pubsub_subscriptions ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_subscriptions' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end; _ -> - ok - end; - _ -> - ok - end. + ok + end. %% ------------------------------------------------------------------- %% @private @@ -245,27 +263,27 @@ migrate_pubsub_subscriptions([H|_]) -> -spec migrate_pubsub_items(Data :: list()) -> Result :: ok. migrate_pubsub_items([]) -> - ok; + ok; migrate_pubsub_items([H|_]) -> - {IID, NodeIdx} = H#pubsub_item.itemid, - {CTime, CJID} = H#pubsub_item.creation, - [Payload] = H#pubsub_item.payload, - CT = to_unixtime(CTime), - JidBIn = jid:to_binary(CJID), - JID = jid:from_binary(JidBIn), - XMLB = <<"", (exml:to_binary(Payload))/binary, "">>, - Cols = ["nidx", "itemid", "created_luser", "created_lserver", "created_at", - "modified_luser", "modified_lserver", "modified_lresource", "modified_at", "payload"], - Vals = [NodeIdx, IID, JID#jid.luser, JID#jid.lserver, CT, JID#jid.lserver, - JID#jid.lserver, JID#jid.lresource, CT, XMLB], - Q = ["INSERT INTO pubsub_items ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'pubsub_items' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {IID, NodeIdx} = H#pubsub_item.itemid, + {CTime, CJID} = H#pubsub_item.creation, + [Payload] = H#pubsub_item.payload, + CT = to_unixtime(CTime), + JidBIn = jid:to_binary(CJID), + JID = jid:from_binary(JidBIn), + XMLB = <<"", (exml:to_binary(Payload))/binary, "">>, + Cols = ["nidx", "itemid", "created_luser", "created_lserver", "created_at", + "modified_luser", "modified_lserver", "modified_lresource", "modified_at", "payload"], + Vals = [NodeIdx, IID, JID#jid.luser, JID#jid.lserver, CT, JID#jid.lserver, + JID#jid.lserver, JID#jid.lresource, CT, XMLB], + Q = ["INSERT INTO pubsub_items ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'pubsub_items' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -276,19 +294,19 @@ migrate_pubsub_items([H|_]) -> -spec migrate_vcard(Data :: list()) -> Result :: ok. migrate_vcard([]) -> - ok; + ok; migrate_vcard([H|_]) -> - {Luser, Lserver} = H#vcard.us, - Cols = ["username", "server", "vcard"], - Vals = [Luser, Lserver, exml:to_binary(H#vcard.vcard)], - Q = ["INSERT INTO vcard ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'vcard' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Luser, Lserver} = H#vcard.us, + Cols = ["username", "server", "vcard"], + Vals = [Luser, Lserver, exml:to_binary(H#vcard.vcard)], + Q = ["INSERT INTO vcard ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'vcard' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -299,27 +317,27 @@ migrate_vcard([H|_]) -> -spec migrate_vcard_search(Data :: list()) -> Result :: ok. migrate_vcard_search([]) -> - ok; + ok; migrate_vcard_search([H|_]) -> - {Luser, Lserver} = H#vcard_search.us, - Cols = ["username", "lusername", "server", "fn", "lfn", "family", "lfamily", "given", - "lgiven", "middle", "lmiddle", "nickname", "lnickname", "bday", "lbday", "ctry", - "lctry", "locality", "llocality", "email", "lemail", "orgname", "lorgname", "orgunit", - "lorgunit"], - Vals = [Luser, H#vcard_search.luser, Lserver, H#vcard_search.fn, H#vcard_search.lfn, - H#vcard_search.family, H#vcard_search.lfamily, H#vcard_search.given, H#vcard_search.lgiven, - H#vcard_search.middle, H#vcard_search.lmiddle, H#vcard_search.nickname, H#vcard_search.lnickname, - H#vcard_search.bday, H#vcard_search.lbday, H#vcard_search.ctry, H#vcard_search.lctry, - H#vcard_search.locality, H#vcard_search.llocality, H#vcard_search.email, H#vcard_search.lemail, - H#vcard_search.orgname, H#vcard_search.lorgname, H#vcard_search.orgunit, H#vcard_search.lorgunit], - Q = ["INSERT INTO vcard_search ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'vcard_search' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Luser, Lserver} = H#vcard_search.us, + Cols = ["username", "lusername", "server", "fn", "lfn", "family", "lfamily", "given", + "lgiven", "middle", "lmiddle", "nickname", "lnickname", "bday", "lbday", "ctry", + "lctry", "locality", "llocality", "email", "lemail", "orgname", "lorgname", "orgunit", + "lorgunit"], + Vals = [Luser, H#vcard_search.luser, Lserver, H#vcard_search.fn, H#vcard_search.lfn, + H#vcard_search.family, H#vcard_search.lfamily, H#vcard_search.given, H#vcard_search.lgiven, + H#vcard_search.middle, H#vcard_search.lmiddle, H#vcard_search.nickname, H#vcard_search.lnickname, + H#vcard_search.bday, H#vcard_search.lbday, H#vcard_search.ctry, H#vcard_search.lctry, + H#vcard_search.locality, H#vcard_search.llocality, H#vcard_search.email, H#vcard_search.lemail, + H#vcard_search.orgname, H#vcard_search.lorgname, H#vcard_search.orgunit, H#vcard_search.lorgunit], + Q = ["INSERT INTO vcard_search ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'vcard_search' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -330,25 +348,25 @@ migrate_vcard_search([H|_]) -> -spec migrate_users(Data :: list()) -> Result :: ok. migrate_users([]) -> - ok; + ok; migrate_users([H|_]) -> - Password = case ejabberd_auth_internal:get_password(H) of - P when is_binary(P) -> - P; - P -> - mongoose_scram:serialize(P) - end, - {Luser, _} = ejabberd_auth_internal:get_us(H), - Cols = ["username", "password"], - Vals = [Luser, Password], - Q = ["INSERT INTO users ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'users' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + Password = case ejabberd_auth_internal:get_password(H) of + P when is_binary(P) -> + P; + P -> + mongoose_scram:serialize(P) + end, + {Luser, _} = ejabberd_auth_internal:get_us(H), + Cols = ["username", "password"], + Vals = [Luser, Password], + Q = ["INSERT INTO users ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'users' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -359,22 +377,22 @@ migrate_users([H|_]) -> -spec migrate_event_pusher_push_subscription(Data :: list()) -> Result :: ok. migrate_event_pusher_push_subscription([]) -> - ok; + ok; migrate_event_pusher_push_subscription([H|_]) -> - {Luser, Lserver} = mod_event_pusher_push_mnesia:get_user_jid(H), - Cols = ["owner_jid", "node", "pubsub_jid", "form", "created_at"], - Vals = [<>, mod_event_pusher_push_mnesia:get_pubsub_node(H), - jid:to_binary(mod_event_pusher_push_mnesia:get_pubsub_jid(H)), - jiffy:encode({mod_event_pusher_push_mnesia:get_form(H)}), - integer_to_binary(os:system_time(micro_seconds))], - Q = ["INSERT INTO event_pusher_push_subscription ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'event_pusher_push_subscription' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Luser, Lserver} = mod_event_pusher_push_mnesia:get_user_jid(H), + Cols = ["owner_jid", "node", "pubsub_jid", "form", "created_at"], + Vals = [<>, mod_event_pusher_push_mnesia:get_pubsub_node(H), + jid:to_binary(mod_event_pusher_push_mnesia:get_pubsub_jid(H)), + jiffy:encode({mod_event_pusher_push_mnesia:get_form(H)}), + integer_to_binary(os:system_time(micro_seconds))], + Q = ["INSERT INTO event_pusher_push_subscription ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'event_pusher_push_subscription' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -385,21 +403,21 @@ migrate_event_pusher_push_subscription([H|_]) -> -spec migrate_rosterusers(Data :: list()) -> Result :: ok. migrate_rosterusers([]) -> - ok; + ok; migrate_rosterusers([H|_]) -> - {Luser, _} = H#roster.us, - Cols = ["username", "jid", "nick", "askmessage", "subscription", "ask", "server", "subscribe", "type"], - Vals = [Luser, jid:to_binary(H#roster.jid), H#roster.name, H#roster.askmessage, - subscription_to_char(H#roster.subscription), ask_to_char(H#roster.ask), - "", "", "item"], % @TODO what should be "server" - char(1)??? and what should be "subscribe" and "type" - Q = ["INSERT INTO rosterusers ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'rosterusers' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Luser, _} = H#roster.us, + Cols = ["username", "jid", "nick", "askmessage", "subscription", "ask", "server", "subscribe", "type"], + Vals = [Luser, jid:to_binary(H#roster.jid), H#roster.name, H#roster.askmessage, + subscription_to_char(H#roster.subscription), ask_to_char(H#roster.ask), + "", "", "item"], % @TODO what should be "server" - char(1)??? and what should be "subscribe" and "type" + Q = ["INSERT INTO rosterusers ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'rosterusers' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -410,19 +428,19 @@ migrate_rosterusers([H|_]) -> -spec migrate_roster_version(Data :: list()) -> Result :: ok. migrate_roster_version([]) -> - ok; + ok; migrate_roster_version([H|_]) -> - {Luser, _} = H#roster_version.us, - Cols = ["username", "version"], - Vals = [Luser, H#roster_version.version], - Q = ["INSERT INTO roster_version ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'roster_version' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Luser, _} = H#roster_version.us, + Cols = ["username", "version"], + Vals = [Luser, H#roster_version.version], + Q = ["INSERT INTO roster_version ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'roster_version' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -433,22 +451,22 @@ migrate_roster_version([H|_]) -> -spec migrate_rostergroups(Data :: list()) -> Result :: ok. migrate_rostergroups([]) -> - ok; + ok; migrate_rostergroups([H|_]) -> - {Luser, _} = H#roster.us, - Cols = ["username", "jid", "grp"], - Vals = [Luser, jid:to_binary(H#roster.jid)], - _ = [begin - Q = ["INSERT INTO rostergroups ", expand_sql_vals(Cols, Vals ++ [Group]), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'rostergroups' has error: ~p for values ~p", [Reason, Vals ++ Group]); - _ -> - ok - end - end || Group <- H#roster.groups], - ok. + {Luser, _} = H#roster.us, + Cols = ["username", "jid", "grp"], + Vals = [Luser, jid:to_binary(H#roster.jid)], + _ = [begin + Q = ["INSERT INTO rostergroups ", expand_sql_vals(Cols, Vals ++ [Group]), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'rostergroups' has error: ~p for values ~p", [Reason, Vals ++ Group]); + _ -> + ok + end + end || Group <- H#roster.groups], + ok. %% ------------------------------------------------------------------- %% @private @@ -459,19 +477,19 @@ migrate_rostergroups([H|_]) -> -spec migrate_last(Data :: list()) -> Result :: ok. migrate_last([]) -> - ok; + ok; migrate_last([H|_]) -> - {Luser, _} = H#last_activity.us, - Cols = ["username", "seconds", "state"], - Vals = [Luser, H#last_activity.timestamp, H#last_activity.status], - Q = ["INSERT INTO last ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'last' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {Luser, _} = H#last_activity.us, + Cols = ["username", "seconds", "state"], + Vals = [Luser, H#last_activity.timestamp, H#last_activity.status], + Q = ["INSERT INTO last ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'last' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -482,19 +500,19 @@ migrate_last([H|_]) -> -spec migrate_private_storage(Data :: list()) -> Result :: ok. migrate_private_storage([]) -> - ok; + ok; migrate_private_storage([H|_]) -> - {LUser, _, NS} = mod_private_mnesia:get_usns(H), - Cols = ["username", "namespace", "data"], - Vals = [LUser, NS, exml:to_binary(mod_private_mnesia:get_xml(H))], - Q = ["INSERT INTO private_storage ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'private_storage' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {LUser, _, NS} = mod_private_mnesia:get_usns(H), + Cols = ["username", "namespace", "data"], + Vals = [LUser, NS, exml:to_binary(mod_private_mnesia:get_xml(H))], + Q = ["INSERT INTO private_storage ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'private_storage' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -505,20 +523,20 @@ migrate_private_storage([H|_]) -> -spec migrate_offline_message(Data :: list()) -> Result :: ok. migrate_offline_message([]) -> - ok; + ok; migrate_offline_message([H|_]) -> - {LUser, LServer} = H#offline_msg.us, - Cols = ["timestamp", "server", "username", "from_jid", "packet"], - Vals = [integer_to_binary(to_unixtime(H#offline_msg.timestamp)), LServer, - LUser, jid:to_binary(H#offline_msg.from), exml:to_binary(H#offline_msg.packet)], - Q = ["INSERT INTO offline_message ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'offline_message' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + {LUser, LServer} = H#offline_msg.us, + Cols = ["timestamp", "server", "username", "from_jid", "packet"], + Vals = [integer_to_binary(to_unixtime(H#offline_msg.timestamp)), LServer, + LUser, jid:to_binary(H#offline_msg.from), exml:to_binary(H#offline_msg.packet)], + Q = ["INSERT INTO offline_message ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'offline_message' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -529,27 +547,27 @@ migrate_offline_message([H|_]) -> -spec migrate_muc_light_rooms(Data :: list()) -> Result :: ok. migrate_muc_light_rooms([]) -> - ok; + ok; migrate_muc_light_rooms([H|_]) -> - {LUser, LServer} = mod_muc_light_db_mnesia:get_room(H), - ID = case mongoose_rdbms:sql_query(?MYNAME, "SELECT MAX(ID) from muc_light_rooms;") of - {selected, [{null}]} -> - 1; - {selected, [{N}]} when is_integer(N) -> - N + 1 - end, - Cols = ["id", "luser", "lserver", "version"], - Vals = [ID, LUser, LServer, mod_muc_light_db_mnesia:get_room_version(H)], - Q = ["INSERT INTO muc_light_rooms ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'muc_light_rooms' has error: ~p for values ~p", [Reason, Vals]); - _ -> - migrate_muc_light_config(mod_muc_light_db_mnesia:get_room_config(H), ID), - _ = [migrate_muc_light_occupants(ID, U, S, Aff) || {{U, S}, Aff} <- mod_muc_light_db_mnesia:get_room_aff_users(H)], - ok - end. + {LUser, LServer} = mod_muc_light_db_mnesia:get_room(H), + ID = case mongoose_rdbms:sql_query(?MYNAME, "SELECT MAX(ID) from muc_light_rooms;") of + {selected, [{null}]} -> + 1; + {selected, [{N}]} when is_integer(N) -> + N + 1 + end, + Cols = ["id", "luser", "lserver", "version"], + Vals = [ID, LUser, LServer, mod_muc_light_db_mnesia:get_room_version(H)], + Q = ["INSERT INTO muc_light_rooms ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'muc_light_rooms' has error: ~p for values ~p", [Reason, Vals]); + _ -> + migrate_muc_light_config(mod_muc_light_db_mnesia:get_room_config(H), ID), + _ = [migrate_muc_light_occupants(ID, U, S, Aff) || {{U, S}, Aff} <- mod_muc_light_db_mnesia:get_room_aff_users(H)], + ok + end. %% ------------------------------------------------------------------- %% @private @@ -561,15 +579,15 @@ migrate_muc_light_rooms([H|_]) -> -spec migrate_muc_light_config(Data :: list(), RoomID :: integer()) -> ok. migrate_muc_light_config([{_, Name}, {_, Subject}], RoomID) -> - Cols = ["room_id", "opt", "val"], - Vals = [RoomID, Name, Subject], - Q = ["INSERT INTO muc_light_config ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'muc_light_config' has error: ~p for values ~p", [Reason, Vals]); - _ -> - ok - end. + Cols = ["room_id", "opt", "val"], + Vals = [RoomID, Name, Subject], + Q = ["INSERT INTO muc_light_config ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'muc_light_config' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end. %% ------------------------------------------------------------------- %% @private @@ -581,28 +599,28 @@ migrate_muc_light_config([{_, Name}, {_, Subject}], RoomID) -> -spec migrate_muc_light_occupants(ID :: integer(), U :: binary(), S :: binary(), A :: atom()) -> ok. migrate_muc_light_occupants(ID, LUser, LServer, Aff) -> - AffDB = case Aff of - owner -> - 1; - member -> - 2; - _ -> - 0 - end, - case AffDB of - 0 -> - ok; - AffDB -> - Cols = ["room_id", "luser", "lserver", "aff"], - Vals = [ID, LUser, LServer, AffDB], - Q = ["INSERT INTO muc_light_occupants ", expand_sql_vals(Cols, Vals), ";"], - case mongoose_rdbms:sql_query(?MYNAME, Q) of - {error, Reason} -> - ?WARNING_MSG("The SQL for 'migrate_muc_light_occupants' has error: ~p for values ~p", [Reason, Vals]); + AffDB = case Aff of + owner -> + 1; + member -> + 2; _ -> - ok - end - end. + 0 + end, + case AffDB of + 0 -> + ok; + AffDB -> + Cols = ["room_id", "luser", "lserver", "aff"], + Vals = [ID, LUser, LServer, AffDB], + Q = ["INSERT INTO muc_light_occupants ", expand_sql_vals(Cols, Vals), ";"], + case mongoose_rdbms:sql_query(?MYNAME, Q) of + {error, Reason} -> + ?WARNING_MSG("The SQL for 'migrate_muc_light_occupants' has error: ~p for values ~p", [Reason, Vals]); + _ -> + ok + end + end. %% ------------------------------------------------------------------- %% @private @@ -613,7 +631,7 @@ migrate_muc_light_occupants(ID, LUser, LServer, Aff) -> -spec to_unixtime(Data :: tuple()) -> Result :: integer(). to_unixtime({Mega, Sec, Micro}) -> - Mega * 1000000 * 1000000 + Sec * 1000000 + Micro. + Mega * 1000000 * 1000000 + Sec * 1000000 + Micro. %% ------------------------------------------------------------------- %% @private @@ -624,15 +642,15 @@ to_unixtime({Mega, Sec, Micro}) -> -spec get_data(Step :: atom(), Name :: atom() | tuple()) -> Result :: {ok, any(), any()} | {error, list()}. get_data(first, Name) -> - try - Key = mnesia:dirty_first(Name), - {ok, mnesia:dirty_read(Name, Key), mnesia:dirty_next(Name, Key)} - catch Class:Exception -> - {error, io_lib:format("Migration call: [~p:~p/1] catched ~p:~p", [?MODULE, get_data, Class, Exception])} - end; + try + Key = mnesia:dirty_first(Name), + {ok, mnesia:dirty_read(Name, Key), mnesia:dirty_next(Name, Key)} + catch Class:Exception -> + {error, io_lib:format("Migration call: [~p:~p/1] catched ~p:~p", [?MODULE, get_data, Class, Exception])} + end; get_data(next, {Name, Key}) -> - {ok, mnesia:dirty_read(Name, Key), mnesia:dirty_next(Name, Key)}. + {ok, mnesia:dirty_read(Name, Key), mnesia:dirty_next(Name, Key)}. %% ------------------------------------------------------------------- %% @private @@ -643,8 +661,8 @@ get_data(next, {Name, Key}) -> -spec expand_sql_vals(Cols :: list(), Vals :: list()) -> Q :: list(). expand_sql_vals(Cols, Vals) -> - EVals = [escape(V) || V <- Vals], - ["(", join(Cols, ", "), ") VALUES (", join(EVals, ", "), ")"]. + EVals = [escape(V) || V <- Vals], + ["(", join(Cols, ", "), ") VALUES (", join(EVals, ", "), ")"]. %% ------------------------------------------------------------------- %% @private @@ -655,10 +673,10 @@ expand_sql_vals(Cols, Vals) -> -spec join(list(), list()) -> list(). join([], _Sep) -> - []; + []; join([H|T], Sep) -> - [H, [[Sep, X] || X <- T]]. + [H, [[Sep, X] || X <- T]]. %% ------------------------------------------------------------------- %% @private @@ -669,13 +687,13 @@ join([H|T], Sep) -> -spec escape(Data :: any()) -> Res :: binary() | list() | any(). escape(Data) when is_integer(Data) -> - integer_to_binary(Data); + integer_to_binary(Data); escape(Data) when is_binary(Data); is_list(Data) -> - [$', mongoose_rdbms:escape_characters(Data), $']; + [$', mongoose_rdbms:escape_characters(Data), $']; escape(Data) -> - Data. + Data. %% ------------------------------------------------------------------- %% @private @@ -686,18 +704,18 @@ escape(Data) -> -spec subscription_to_char(Data :: atom()) -> Result :: binary(). subscription_to_char(Subs) -> - case Subs of - both -> - <<"B">>; - to -> - <<"T">>; - from -> - <<"F">>; - remove -> - <<"R">>; - none -> - <<"N">> - end. + case Subs of + both -> + <<"B">>; + to -> + <<"T">>; + from -> + <<"F">>; + remove -> + <<"R">>; + none -> + <<"N">> + end. %% ------------------------------------------------------------------- %% @private @@ -708,17 +726,17 @@ subscription_to_char(Subs) -> -spec ask_to_char(Data :: atom()) -> Result :: binary(). ask_to_char(Ask) -> - case Ask of - subscribe -> - <<"S">>; - unsubscribe -> - <<"U">>; - both -> - <<"B">>; - out -> - <<"O">>; - in -> - <<"I">>; - _ -> - <<"N">> - end. + case Ask of + subscribe -> + <<"S">>; + unsubscribe -> + <<"U">>; + both -> + <<"B">>; + out -> + <<"O">>; + in -> + <<"I">>; + _ -> + <<"N">> + end. From 3cd6b42d74ebdcff2ac1b7327e220d332b2203c8 Mon Sep 17 00:00:00 2001 From: vkatsuba Date: Tue, 22 Dec 2020 18:04:41 +0200 Subject: [PATCH 3/4] Mnesia migration: Added CT for migrate_pubsub_nodes, migrate_pubsub_subscriptions, migrate_pubsub_affiliations, migrate_pubsub_items, migrate_users, migrate_vcard_search, migrate_vcard, migrate_event_pusher_push_subscription --- big_tests/tests/mnesia_migration_SUITE.erl | 201 +++++++++++++++--- .../service_admin_extra_migration.erl | 5 +- src/pubsub/mod_pubsub_db_mnesia.erl | 12 +- 3 files changed, 185 insertions(+), 33 deletions(-) diff --git a/big_tests/tests/mnesia_migration_SUITE.erl b/big_tests/tests/mnesia_migration_SUITE.erl index b5c8a0b41c4..bcf8a5fbea1 100644 --- a/big_tests/tests/mnesia_migration_SUITE.erl +++ b/big_tests/tests/mnesia_migration_SUITE.erl @@ -15,13 +15,31 @@ %%% ================================================================== -define(RPC_MIGRATE(Act), mongoose_helper:successful_rpc(service_admin_extra_migration, migrate, [<<"mnesia">>, <<"rdbms">>, Act])). +-define(JID, jid:binary_to_bare(<<"migration_user@localhost/test">>)). +-define(NODE_NAME, <<"migration_node">>). +-define(USERNAME, <<"migration_user">>). +-define(NODE_HOST, ct:get_config({hosts, mim, domain})). +-define(BASE_NODE, {pubsub_node, {?NODE_HOST, ?NODE_NAME}, undefined, [], <<"flat">>, [?JID], []}). +-define(BASE_ITEM(Nidx), {pubsub_item, {?NODE_NAME, Nidx}, {erlang:timestamp(), ?JID}, {erlang:timestamp(),?JID}, ?JID, [{xmlcdata, []}]}). +-define(BASE_VCARD, {xmlel,<<"vCard">>, + [{<<"xmlns">>,<<"vcard-temp">>}], + [{xmlel,<<"NICKNAME">>,[],[{xmlcdata, ?USERNAME}]}, + {xmlel,<<"TEL">>,[], + [{xmlel,<<"HOME">>,[],[]}, + {xmlel,<<"VOICE">>,[],[]}, + {xmlel,<<"NUMBER">>,[], + [{xmlcdata,<<"+00000000000">>}]}]}]}). +-define(BASE_SEARCH_VCARD, {vcard_search, {<<"migration_user">>, <<"localhost">>}, <>, ?USERNAME, <<"fn">>, <<"lfn">>, <<"family">>, + <<"lfamily">>, <<"given">>, <<"lgiven">>, <<"middle">>, <<"lmiddle">>, <<"nickname">>, <<"lnickname">>, <<"bday">>, <<"lbday">>, + <<"ctry">>, <<"lctry">>, <<"locality">>, <<"llocality">>, <<"email">>, <<"lemail">>, <<"orgname">>, <<"lorgname">>, <<"orgunit">>, + <<"lorgunit">>}). %%-------------------------------------------------------------------- %% Suite configuration %%-------------------------------------------------------------------- all() -> - case (not ct_helper:is_ct_running()) orelse mongoose_helper:is_rdbms_enabled(ct:get_config({hosts, mim, domain})) of + case (not ct_helper:is_ct_running()) orelse mongoose_helper:is_rdbms_enabled(?NODE_HOST) of true -> tests(); false -> @@ -63,9 +81,18 @@ suite() -> escalus:suite(). init_per_suite(Config) -> + ok = mnesia:create_schema([node()]), + ok = mnesia:start(), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, start, []), + ok = mongoose_helper:successful_rpc(ejabberd_auth_internal, start, [?NODE_HOST]), + ok = mongoose_helper:successful_rpc(mod_vcard_mnesia, init, [<<>>, <<>>]), + ok = mongoose_helper:successful_rpc(mod_event_pusher_push_mnesia, init, [<<>>, <<>>]), + {ok, _} = application:ensure_all_started(jid), escalus:init_per_suite(Config). end_per_suite(Config) -> + ok = mongoose_helper:successful_rpc(ejabberd_auth_internal, stop, [?NODE_HOST]), + _ = mnesia:stop(), escalus_fresh:clean(), escalus:end_per_suite(Config). @@ -82,68 +109,143 @@ end_per_testcase(CaseName, Config) -> escalus:end_per_testcase(CaseName, Config). %%% ================================================================== -%%% Test case for migrate pubsub_nodes @TODO +%%% Test case for migrate pubsub_nodes %%% ================================================================== -migrate_pubsub_nodes(Config) -> - R = ?RPC_MIGRATE(<<"pubsub_nodes">>), - ct:comment("TEST CASE ~p", [{?FUNCTION_NAME, R}]). +migrate_pubsub_nodes(_Config) -> + Nidx = create_migration_node(), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_nodes">>), + SqlData = #{table => <<"pubsub_nodes">>, where => <<"name='", ?NODE_NAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_nodes' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_nodes' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate pubsub_subscriptions @TODO +%%% Test case for migrate pubsub_subscriptions %%% ================================================================== -migrate_pubsub_subscriptions(Config) -> - ?RPC_MIGRATE(<<"pubsub_subscriptions">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_pubsub_subscriptions(_Config) -> + Nidx = create_migration_node(), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction, [#{name => add_subscription, args => [Nidx, ?JID, 'subscribed', <<"0000-0000-0000000">>, []]}]), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_subscriptions">>), + SqlData = #{table => <<"pubsub_subscriptions">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_subscriptions' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_subscriptions' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate pubsub_affiliations @TODO +%%% Test case for migrate pubsub_affiliations %%% ================================================================== -migrate_pubsub_affiliations(Config) -> - ?RPC_MIGRATE(<<"pubsub_affiliations">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_pubsub_affiliations(_Config) -> + Nidx = create_migration_node(), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_affiliations">>), + SqlData = #{table => <<"pubsub_affiliations">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_affiliations' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_affiliations' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate pubsub_items @TODO +%%% Test case for migrate pubsub_items %%% ================================================================== -migrate_pubsub_items(Config) -> - ?RPC_MIGRATE(<<"pubsub_items">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_pubsub_items(_Config) -> + Nidx = create_migration_node(), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction, [#{name => add_item, args => [Nidx, ?JID, ?BASE_ITEM(Nidx)]}]), + {ok, _} = ?RPC_MIGRATE(<<"pubsub_items">>), + SqlData = #{table => <<"pubsub_items">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of + {selected, [{Nidx}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'pubsub_items' is successful for Nidx: ~p", [Nidx]); + Any -> + ct:fail("Unexpected result of 'pubsub_items' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate users @TODO +%%% Test case for migrate users %%% ================================================================== -migrate_users(Config) -> - ?RPC_MIGRATE(<<"users">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_users(_Config) -> + ok = mongoose_helper:successful_rpc(ejabberd_auth_internal, set_password, [?USERNAME, <<"localhost">>, <<"migration_pass">>]), + {ok, _} = ?RPC_MIGRATE(<<"users">>), + SqlData = #{table => <<"users">>, where => <<"username='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"username">>}) of + {selected, [{?USERNAME}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'users' is successful for Username: ~p", [?USERNAME]); + Any -> + ct:fail("Unexpected result of 'users' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate vcard_search @TODO +%%% Test case for migrate vcard_search %%% ================================================================== -migrate_vcard_search(Config) -> - ?RPC_MIGRATE(<<"vcard_search">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). +migrate_vcard_search(_Config) -> + ok = set_vcard(), + {ok, _} = ?RPC_MIGRATE(<<"vcard_search">>), + SqlData = #{table => <<"vcard_search">>, where => <<"username='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"username">>}) of + {selected, [{?USERNAME}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'vcard_search' is successful for Username: ~p", [?USERNAME]); + Any -> + ct:fail("Unexpected result of 'vcard_search' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate vcard @TODO +%%% Test case for migrate vcard %%% ================================================================== migrate_vcard(Config) -> - ?RPC_MIGRATE(<<"vcard">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + ok = set_vcard(), + {ok, _} = ?RPC_MIGRATE(<<"vcard">>), + SqlData = #{table => <<"vcard">>, where => <<"username='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"username">>}) of + {selected, [{?USERNAME}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'vcard' is successful for Username: ~p", [?USERNAME]); + Any -> + ct:fail("Unexpected result of 'vcard' migration ~p~n", [Any]) + end. %%% ================================================================== -%%% Test case for migrate event_pusher_push_subscription @TODO +%%% Test case for migrate event_pusher_push_subscription %%% ================================================================== migrate_event_pusher_push_subscription(Config) -> - ?RPC_MIGRATE(<<"event_pusher_push_subscription">>), - ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + PubsubNode = <<"migration_pub_sub_node">>, + ok = mongoose_helper:successful_rpc(mod_event_pusher_push_mnesia, enable, [?JID, ?JID, PubsubNode, [{<<"name">>, <<"value">>}]]), + {ok, _} = ?RPC_MIGRATE(<<"event_pusher_push_subscription">>), + SqlData = #{table => <<"event_pusher_push_subscription">>, where => <<"node='", PubsubNode/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"node">>}) of + {selected, [{PubsubNode}]} -> + {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), + _ = clear_tables(), + ct:comment("Migration of 'event_pusher_push_subscription' is successful for Node: ~p", [PubsubNode]); + Any -> + ct:fail("Unexpected result of 'event_pusher_push_subscription' migration ~p~n", [Any]) + end. %%% ================================================================== %%% Test case for migrate rosterusers @TODO @@ -208,3 +310,40 @@ migrate_muc_light_rooms(Config) -> migrate_all(Config) -> ?RPC_MIGRATE(<<"all">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). + +%%% ================================================================== +%%% Internal functions +%%% ================================================================== + +sql_query(Query) -> + slow_rpc(mongoose_rdbms, sql_query, [?NODE_HOST, Query]). + +slow_rpc(M, F, A) -> + Node = ct:get_config({hosts, mim, node}), + Cookie = escalus_ct:get_config(ejabberd_cookie), + Res = escalus_rpc:call(Node, M, F, A, timer:seconds(30), Cookie), + case Res of + {badrpc, timeout} -> + {badrpc, {timeout, M, F}}; + _ -> + Res + end. + +sql_to_rdbms(#{act := Act, column := Column, table := Table, where := Where}) -> + SelectQuery = <>, + SelectResult = sql_query(SelectQuery). + +clear_tables() -> + Tables = [pubsub_state, pubsub_item, pubsub_node, + pubsub_subscription, pubsub_subnode, + passwd, reg_users_counter, + vcard, vcard_search, push_subscription], + _ = [mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, clear_table, [T]) || T <- Tables]. + +create_migration_node() -> + {ok, Nidx} = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction,[#{name => set_node, args => [?BASE_NODE]}]), + ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction,[#{name => create_node, args => [Nidx, ?JID]}]), + Nidx. + +set_vcard() -> + mongoose_helper:successful_rpc(mod_vcard_mnesia, set_vcard, [?USERNAME, <<"localhost">>, ?BASE_VCARD, ?BASE_SEARCH_VCARD]). diff --git a/src/admin_extra/service_admin_extra_migration.erl b/src/admin_extra/service_admin_extra_migration.erl index 726afc101ca..f0fb1e0c48c 100644 --- a/src/admin_extra/service_admin_extra_migration.erl +++ b/src/admin_extra/service_admin_extra_migration.erl @@ -157,6 +157,9 @@ try_migrate({Act, Table}) when is_atom(Table) -> try_migrate({Act, Table, TableData}) -> case TableData of + {ok, [_] = Data, '$end_of_table'} -> + ?MODULE:(list_to_existing_atom("migrate_" ++ binary_to_list(Act)))(Data), + {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; {ok, _, '$end_of_table'} -> {ok, io_lib:format("Completed the migration of the table: '~s'", [Act])}; {ok, Data, NextKey} -> @@ -276,7 +279,7 @@ migrate_pubsub_items([H|_]) -> Cols = ["nidx", "itemid", "created_luser", "created_lserver", "created_at", "modified_luser", "modified_lserver", "modified_lresource", "modified_at", "payload"], Vals = [NodeIdx, IID, JID#jid.luser, JID#jid.lserver, CT, JID#jid.lserver, - JID#jid.lserver, JID#jid.lresource, CT, XMLB], + JID#jid.lserver, JID#jid.lresource, CT, XMLB], Q = ["INSERT INTO pubsub_items ", expand_sql_vals(Cols, Vals), ";"], case mongoose_rdbms:sql_query(?MYNAME, Q) of {error, Reason} -> diff --git a/src/pubsub/mod_pubsub_db_mnesia.erl b/src/pubsub/mod_pubsub_db_mnesia.erl index a4eeba23d71..16c66d2420a 100644 --- a/src/pubsub/mod_pubsub_db_mnesia.erl +++ b/src/pubsub/mod_pubsub_db_mnesia.erl @@ -17,7 +17,7 @@ -export([start/0, stop/0]). % Funs execution --export([transaction/2, dirty/2]). +-export([clear_table/1, transaction/1, transaction/2, dirty/2]). % Direct #pubsub_state access -export([del_node/1, get_state/2, get_states/1, get_states_by_lus/1, get_states_by_bare/1, @@ -134,6 +134,16 @@ maybe_fill_subnode_table(_Other) -> %% ------------------------ Fun execution ------------------------ +-spec clear_table(atom()) -> + {atomic, ok} | {aborted, term()}. +clear_table(Tab) -> + mnesia:clear_table(Tab). + +-spec transaction(map()) -> + {result | error, any()}. +transaction(#{name := Name, args := Args}) -> + transaction(fun() -> apply(?MODULE, Name, Args) end, #{}). + -spec transaction(Fun :: fun(() -> {result | error, any()}), ErrorDebug :: map()) -> {result | error, any()}. From 73f4ce098b36b538e97c18748139fc245e23c523 Mon Sep 17 00:00:00 2001 From: vkatsuba Date: Thu, 24 Dec 2020 12:45:34 +0200 Subject: [PATCH 4/4] Mnesia migration: Fix logic of CT and CTL for correct MSSQL migration --- big_tests/tests/mnesia_migration_SUITE.erl | 58 +++++++++---------- .../service_admin_extra_migration.erl | 21 ++++--- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/big_tests/tests/mnesia_migration_SUITE.erl b/big_tests/tests/mnesia_migration_SUITE.erl index bcf8a5fbea1..d87586fee66 100644 --- a/big_tests/tests/mnesia_migration_SUITE.erl +++ b/big_tests/tests/mnesia_migration_SUITE.erl @@ -113,14 +113,14 @@ end_per_testcase(CaseName, Config) -> %%% ================================================================== migrate_pubsub_nodes(_Config) -> - Nidx = create_migration_node(), + _ = create_migration_node(), {ok, _} = ?RPC_MIGRATE(<<"pubsub_nodes">>), SqlData = #{table => <<"pubsub_nodes">>, where => <<"name='", ?NODE_NAME/binary, "'">>}, - case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of - {selected, [{Nidx}]} -> + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"name">>}) of + {selected, [{?NODE_NAME}]} -> {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), _ = clear_tables(), - ct:comment("Migration of 'pubsub_nodes' is successful for Nidx: ~p", [Nidx]); + ct:comment("Migration of 'pubsub_nodes' is successful for 'name': ~p", [?NODE_NAME]); Any -> ct:fail("Unexpected result of 'pubsub_nodes' migration ~p~n", [Any]) end. @@ -133,12 +133,12 @@ migrate_pubsub_subscriptions(_Config) -> Nidx = create_migration_node(), ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction, [#{name => add_subscription, args => [Nidx, ?JID, 'subscribed', <<"0000-0000-0000000">>, []]}]), {ok, _} = ?RPC_MIGRATE(<<"pubsub_subscriptions">>), - SqlData = #{table => <<"pubsub_subscriptions">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, - case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of - {selected, [{Nidx}]} -> + SqlData = #{table => <<"pubsub_subscriptions">>, where => <<"luser='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"luser">>}) of + {selected, [{?USERNAME}]} -> {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), _ = clear_tables(), - ct:comment("Migration of 'pubsub_subscriptions' is successful for Nidx: ~p", [Nidx]); + ct:comment("Migration of 'pubsub_subscriptions' is successful for 'luser': ~p", [?USERNAME]); Any -> ct:fail("Unexpected result of 'pubsub_subscriptions' migration ~p~n", [Any]) end. @@ -148,14 +148,14 @@ migrate_pubsub_subscriptions(_Config) -> %%% ================================================================== migrate_pubsub_affiliations(_Config) -> - Nidx = create_migration_node(), + _ = create_migration_node(), {ok, _} = ?RPC_MIGRATE(<<"pubsub_affiliations">>), - SqlData = #{table => <<"pubsub_affiliations">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, - case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of - {selected, [{Nidx}]} -> + SqlData = #{table => <<"pubsub_affiliations">>, where => <<"luser='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"luser">>}) of + {selected, [{?USERNAME}]} -> {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), _ = clear_tables(), - ct:comment("Migration of 'pubsub_affiliations' is successful for Nidx: ~p", [Nidx]); + ct:comment("Migration of 'pubsub_affiliations' is successful for 'pubsub_affiliations': ~p", [?USERNAME]); Any -> ct:fail("Unexpected result of 'pubsub_affiliations' migration ~p~n", [Any]) end. @@ -168,12 +168,12 @@ migrate_pubsub_items(_Config) -> Nidx = create_migration_node(), ok = mongoose_helper:successful_rpc(mod_pubsub_db_mnesia, transaction, [#{name => add_item, args => [Nidx, ?JID, ?BASE_ITEM(Nidx)]}]), {ok, _} = ?RPC_MIGRATE(<<"pubsub_items">>), - SqlData = #{table => <<"pubsub_items">>, where => <<"nidx=", (list_to_binary(integer_to_list(Nidx)))/binary>>}, - case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"nidx">>}) of - {selected, [{Nidx}]} -> + SqlData = #{table => <<"pubsub_items">>, where => <<"created_luser='", ?USERNAME/binary, "'">>}, + case sql_to_rdbms(SqlData#{act => <<"SELECT">>, column => <<"created_luser">>}) of + {selected, [{?USERNAME}]} -> {updated, 1} = sql_to_rdbms(SqlData#{act => <<"DELETE">>, column => <<"">>}), _ = clear_tables(), - ct:comment("Migration of 'pubsub_items' is successful for Nidx: ~p", [Nidx]); + ct:comment("Migration of 'pubsub_items' is successful for 'created_luser': ~p", [?USERNAME]); Any -> ct:fail("Unexpected result of 'pubsub_items' migration ~p~n", [Any]) end. @@ -216,7 +216,7 @@ migrate_vcard_search(_Config) -> %%% Test case for migrate vcard %%% ================================================================== -migrate_vcard(Config) -> +migrate_vcard(_Config) -> ok = set_vcard(), {ok, _} = ?RPC_MIGRATE(<<"vcard">>), SqlData = #{table => <<"vcard">>, where => <<"username='", ?USERNAME/binary, "'">>}, @@ -233,7 +233,7 @@ migrate_vcard(Config) -> %%% Test case for migrate event_pusher_push_subscription %%% ================================================================== -migrate_event_pusher_push_subscription(Config) -> +migrate_event_pusher_push_subscription(_Config) -> PubsubNode = <<"migration_pub_sub_node">>, ok = mongoose_helper:successful_rpc(mod_event_pusher_push_mnesia, enable, [?JID, ?JID, PubsubNode, [{<<"name">>, <<"value">>}]]), {ok, _} = ?RPC_MIGRATE(<<"event_pusher_push_subscription">>), @@ -251,7 +251,7 @@ migrate_event_pusher_push_subscription(Config) -> %%% Test case for migrate rosterusers @TODO %%% ================================================================== -migrate_rosterusers(Config) -> +migrate_rosterusers(_Config) -> ?RPC_MIGRATE(<<"rosterusers">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -259,7 +259,7 @@ migrate_rosterusers(Config) -> %%% Test case for migrate roster_version @TODO %%% ================================================================== -migrate_roster_version(Config) -> +migrate_roster_version(_Config) -> ?RPC_MIGRATE(<<"roster_version">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -267,7 +267,7 @@ migrate_roster_version(Config) -> %%% Test case for migrate rostergroups @TODO %%% ================================================================== -migrate_rostergroups(Config) -> +migrate_rostergroups(_Config) -> ?RPC_MIGRATE(<<"rostergroups">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -275,7 +275,7 @@ migrate_rostergroups(Config) -> %%% Test case for migrate last @TODO %%% ================================================================== -migrate_last(Config) -> +migrate_last(_Config) -> ?RPC_MIGRATE(<<"last">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -283,7 +283,7 @@ migrate_last(Config) -> %%% Test case for migrate private_storage @TODO %%% ================================================================== -migrate_private_storage(Config) -> +migrate_private_storage(_Config) -> ?RPC_MIGRATE(<<"private_storage">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -291,7 +291,7 @@ migrate_private_storage(Config) -> %%% Test case for migrate offline_message @TODO %%% ================================================================== -migrate_offline_message(Config) -> +migrate_offline_message(_Config) -> ?RPC_MIGRATE(<<"offline_message">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -299,7 +299,7 @@ migrate_offline_message(Config) -> %%% Test case for migrate muc_light_rooms @TODO %%% ================================================================== -migrate_muc_light_rooms(Config) -> +migrate_muc_light_rooms(_Config) -> ?RPC_MIGRATE(<<"muc_light_rooms">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -307,7 +307,7 @@ migrate_muc_light_rooms(Config) -> %%% Test case for migrate all @TODO %%% ================================================================== -migrate_all(Config) -> +migrate_all(_Config) -> ?RPC_MIGRATE(<<"all">>), ct:comment("TEST CASE ~p", [?FUNCTION_NAME]). @@ -331,7 +331,7 @@ slow_rpc(M, F, A) -> sql_to_rdbms(#{act := Act, column := Column, table := Table, where := Where}) -> SelectQuery = <>, - SelectResult = sql_query(SelectQuery). + sql_query(SelectQuery). clear_tables() -> Tables = [pubsub_state, pubsub_item, pubsub_node, @@ -346,4 +346,4 @@ create_migration_node() -> Nidx. set_vcard() -> - mongoose_helper:successful_rpc(mod_vcard_mnesia, set_vcard, [?USERNAME, <<"localhost">>, ?BASE_VCARD, ?BASE_SEARCH_VCARD]). + mongoose_helper:successful_rpc(mod_vcard_mnesia, set_vcard, [?USERNAME, <<"localhost">>, ?BASE_VCARD, ?BASE_SEARCH_VCARD]). diff --git a/src/admin_extra/service_admin_extra_migration.erl b/src/admin_extra/service_admin_extra_migration.erl index f0fb1e0c48c..b480e018022 100644 --- a/src/admin_extra/service_admin_extra_migration.erl +++ b/src/admin_extra/service_admin_extra_migration.erl @@ -184,8 +184,8 @@ migrate_pubsub_nodes([H|_]) -> {Host, ID} = H#pubsub_node.nodeid, Owners = jiffy:encode([jid:to_binary(O) || O <- H#pubsub_node.owners]), Ops = jiffy:encode({H#pubsub_node.options}), - Cols = ["nidx", "p_key", "name", "type", "owners", "options"], - Vals = [H#pubsub_node.id, Host, ID, H#pubsub_node.type, Owners, Ops], + Cols = ["p_key", "name", "type", "owners", "options"], % "nidx" + Vals = [Host, ID, H#pubsub_node.type, Owners, Ops], % H#pubsub_node.id Q = ["INSERT INTO pubsub_nodes ", expand_sql_vals(Cols, Vals), ";"], case mongoose_rdbms:sql_query(?MYNAME, Q) of {error, Reason} -> @@ -271,15 +271,15 @@ migrate_pubsub_items([]) -> migrate_pubsub_items([H|_]) -> {IID, NodeIdx} = H#pubsub_item.itemid, {CTime, CJID} = H#pubsub_item.creation, - [Payload] = H#pubsub_item.payload, + Payload = H#pubsub_item.payload, CT = to_unixtime(CTime), JidBIn = jid:to_binary(CJID), JID = jid:from_binary(JidBIn), - XMLB = <<"", (exml:to_binary(Payload))/binary, "">>, + XMLB = exml:to_binary(#xmlel{name = <<"item">>, children = Payload}), Cols = ["nidx", "itemid", "created_luser", "created_lserver", "created_at", "modified_luser", "modified_lserver", "modified_lresource", "modified_at", "payload"], Vals = [NodeIdx, IID, JID#jid.luser, JID#jid.lserver, CT, JID#jid.lserver, - JID#jid.lserver, JID#jid.lresource, CT, XMLB], + JID#jid.lserver, JID#jid.lresource, CT, {XMLB, escape_binary}], Q = ["INSERT INTO pubsub_items ", expand_sql_vals(Cols, Vals), ";"], case mongoose_rdbms:sql_query(?MYNAME, Q) of {error, Reason} -> @@ -558,10 +558,12 @@ migrate_muc_light_rooms([H|_]) -> {selected, [{null}]} -> 1; {selected, [{N}]} when is_integer(N) -> - N + 1 + N + 1; + {selected, [{N}]} when is_binary(N) -> + list_to_integer(binary_to_list(N)) + 1 end, - Cols = ["id", "luser", "lserver", "version"], - Vals = [ID, LUser, LServer, mod_muc_light_db_mnesia:get_room_version(H)], + Cols = ["luser", "lserver", "version"], % ---"id" + Vals = [LUser, LServer, mod_muc_light_db_mnesia:get_room_version(H)], % --- ID Q = ["INSERT INTO muc_light_rooms ", expand_sql_vals(Cols, Vals), ";"], case mongoose_rdbms:sql_query(?MYNAME, Q) of {error, Reason} -> @@ -689,6 +691,9 @@ join([H|T], Sep) -> %% ------------------------------------------------------------------- -spec escape(Data :: any()) -> Res :: binary() | list() | any(). +escape({Data, Escape}) -> + mongoose_rdbms_odbc:Escape(Data); + escape(Data) when is_integer(Data) -> integer_to_binary(Data);