1.multi_view_pattern_mining.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import time\n",
    "import math\n",
    "import random\n",
    "import os\n",
    "from collections import Counter\n",
    "import copy\n",
    "import pickle\n",
    "import os \n",
    "import csv\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# namedataset=\"Helpdesk\"\n",
    "# namedataset=\"BPI_Challenge_2012_W_Complete\"\n",
    "# namedataset=\"BPI_Challenge_2012_W\"\n",
    "# namedataset=\"BPI_Challenge_2012_A\"\n",
    "# namedataset=\"BPI_Challenge_2012_O\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hsc/miniconda3/envs/py3.9/lib/python3.9/site-packages/sklearn/cluster/_spectral.py:658: UserWarning: The spectral clustering API has changed. ``fit``now constructs an affinity matrix from data. To use a custom affinity matrix, set ``affinity=precomputed``.\n",
      "  warnings.warn(\n",
      "/home/hsc/miniconda3/envs/py3.9/lib/python3.9/site-packages/sklearn/cluster/_spectral.py:658: UserWarning: The spectral clustering API has changed. ``fit``now constructs an affinity matrix from data. To use a custom affinity matrix, set ``affinity=precomputed``.\n",
      "  warnings.warn(\n",
      "/home/hsc/miniconda3/envs/py3.9/lib/python3.9/site-packages/sklearn/cluster/_spectral.py:658: UserWarning: The spectral clustering API has changed. ``fit``now constructs an affinity matrix from data. To use a custom affinity matrix, set ``affinity=precomputed``.\n",
      "  warnings.warn(\n",
      "/home/hsc/miniconda3/envs/py3.9/lib/python3.9/site-packages/sklearn/cluster/_spectral.py:658: UserWarning: The spectral clustering API has changed. ``fit``now constructs an affinity matrix from data. To use a custom affinity matrix, set ``affinity=precomputed``.\n",
      "  warnings.warn(\n",
      "/home/hsc/miniconda3/envs/py3.9/lib/python3.9/site-packages/sklearn/cluster/_spectral.py:658: UserWarning: The spectral clustering API has changed. ``fit``now constructs an affinity matrix from data. To use a custom affinity matrix, set ``affinity=precomputed``.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "for step in range(5):\n",
    "    train_val_file=\"processed_datasets/\"+namedataset+\"/fold\"+str(step)+\"/train_val_\"+namedataset+\".csv\"\n",
    "    train_file=\"processed_datasets/\"+namedataset+\"/fold\"+str(step)+\"/train_\"+namedataset+\".csv\"\n",
    "    val_file=\"processed_datasets/\"+namedataset+\"/fold\"+str(step)+\"/val_\"+namedataset+\".csv\"\n",
    "    test_file=\"processed_datasets/\"+namedataset+\"/fold\"+str(step)+\"/test_\"+namedataset+\".csv\"\n",
    "    df_test=pd.read_csv(test_file)\n",
    "    try:\n",
    "        df_test.columns = [\"CaseID\", \"Activity\", \"Resource\", \"Timestamp\",\"amount\"]\n",
    "    except:\n",
    "        df_test.columns = [\"CaseID\", \"Activity\", \"Resource\", \"Timestamp\"]\n",
    "\n",
    "    df_train=pd.read_csv(train_val_file)\n",
    "    try:\n",
    "        df_train.columns = [\"CaseID\", \"Activity\", \"Resource\", \"Timestamp\",\"amount\"]\n",
    "    except:\n",
    "        df_train.columns = [\"CaseID\", \"Activity\", \"Resource\", \"Timestamp\"]\n",
    "    df_train.fillna(1, inplace=True)\n",
    "    df_test.fillna(1,inplace=True)\n",
    "    # df_valid.fillna(1,inplace=True)\n",
    "\n",
    "    cont_trace = df_train['CaseID'].value_counts(dropna=False)\n",
    "    cont_test_trace = df_test['CaseID'].value_counts(dropna=False)\n",
    "    max_trace = max(cont_trace)\n",
    "    max_test_trace = max(cont_test_trace)\n",
    "    # max_valid_trace = max(cont_valid_trace)\n",
    "    # real_max=max(max_trace,max_test_trace,max_valid_trace)\n",
    "    real_max=max(max_trace,max_test_trace)\n",
    "\n",
    "    test_listOfResource=df_test[\"Resource\"].unique().tolist()\n",
    "    train_listOfResource=df_train[\"Resource\"].unique().tolist()\n",
    "    # valid_listOfResource=df_valid[\"Resource\"].unique().tolist()\n",
    "\n",
    "    test_resourceset=set(test_listOfResource)\n",
    "    train_resourceset=set(train_listOfResource)\n",
    "    # valid_resourceset=set(valid_listOfaResource)\n",
    "\n",
    "    # all_resource=list(valid_resourceset.union(test_resourceset.union(train_resourceset)))\n",
    "    all_resource=list(test_resourceset.union(train_resourceset))\n",
    "\n",
    "    all_resource.sort()\n",
    "\n",
    "    listOfresourcesInt = list(range(0, len(all_resource)))\n",
    "    resourceMapping=dict(zip(all_resource,listOfresourcesInt))\n",
    "    # 统一活动值\n",
    "\n",
    "    test_listOfEvents=df_test[\"Activity\"].unique().tolist()\n",
    "    train_listOfEvents=df_train[\"Activity\"].unique().tolist()\n",
    "    # valid_listOfEvents=df_valid[\"Activity\"].unique().tolist()\n",
    "\n",
    "    test_eventset=set(test_listOfEvents)\n",
    "    train_eventset=set(train_listOfEvents)\n",
    "    # valid_eventset=set(valid_listOfEvents)\n",
    "\n",
    "    # all_events=list(valid_eventset.union(test_eventset.union(train_eventset)))\n",
    "    all_events=list(test_eventset.union(train_eventset))\n",
    "\n",
    "    all_events.sort()\n",
    "    listOfeventsInt = list(range(0, len(all_events)))\n",
    "    mapping = dict(zip(all_events, listOfeventsInt))\n",
    "\n",
    "    df_test.Activity = [mapping[item] for item in df_test.Activity]\n",
    "    # 将活动名称和id对应\n",
    "    df_train.Activity = [mapping[item] for item in df_train.Activity]\n",
    "    # 将活动名称和id对应\n",
    "\n",
    "    df_test.Resource = [resourceMapping[item] for item in df_test.Resource]\n",
    "    df_train.Resource = [resourceMapping[item] for item in df_train.Resource]\n",
    "\n",
    "    # group by activity, resource and timestamp by caseid\n",
    "    act = df_train.groupby('CaseID', sort=False).agg({'Activity': lambda x: list(x)})\n",
    "    res = df_train.groupby('CaseID', sort=False).agg({'Resource': lambda x: list(x)})\n",
    "    temp = df_train.groupby('CaseID', sort=False).agg({'Timestamp': lambda x: list(x)})\n",
    "    # amount = df_train.groupby('CaseID', sort=False).agg({'Amount': lambda x: list(x)})\n",
    "\n",
    "    from collections import Counter\n",
    "    act,res,temp=act.loc[:,\"Activity\"].values,res.loc[:,\"Resource\"].values,temp.loc[:,\"Timestamp\"].values\n",
    "    res_counter={resource:0 for resource in listOfresourcesInt}\n",
    "    for res_trace in res:\n",
    "        for resource in res_trace:\n",
    "            try:\n",
    "                res_counter[resource]+=1\n",
    "            except:\n",
    "                print(\"what\")\n",
    "    normal_res=[]\n",
    "    abnormal_res=[]\n",
    "    for key in res_counter:\n",
    "        if res_counter[key]>=5:\n",
    "            normal_res.append(key)\n",
    "        else:\n",
    "            abnormal_res.append(key)\n",
    "            \n",
    "    rg=re.compile(\"\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\")\n",
    "    def time_format(ttime):\n",
    "        ttime=rg.search(ttime).group(0)\n",
    "        try:\n",
    "            date_format_str = '%Y/%m/%d %H:%M:%S.%f'\n",
    "            conversion = time.mktime(time.strptime(ttime, date_format_str))\n",
    "        except:\n",
    "            date_format_str = '%Y-%m-%d %H:%M:%S'\n",
    "            conversion = time.mktime(time.strptime(ttime, date_format_str))\n",
    "        return conversion\n",
    "\n",
    "    for i,times in enumerate(temp):\n",
    "        temp[i]=[ time_format(ttime) for ttime in times]\n",
    "\n",
    "    time_difference_lists=[]\n",
    "    for times in temp:\n",
    "        a=np.array(times)[1:]\n",
    "        b=a-np.array(times)[:-1]\n",
    "        time_difference_lists.append(np.concatenate((np.array([0]),b)))\n",
    "\n",
    "    total_time_difference=[]\n",
    "    for difference_list in time_difference_lists:\n",
    "        total_time_difference+= difference_list.tolist()\n",
    "\n",
    "    total_time_difference.sort()\n",
    "\n",
    "    orgdifferencelist=total_time_difference[len(act):]\n",
    "\n",
    "    Q1=orgdifferencelist[int(len(orgdifferencelist)+1)//4]\n",
    "    Q2=orgdifferencelist[int(2*len(orgdifferencelist)+1)//4]\n",
    "    Q3=orgdifferencelist[int(3*len(orgdifferencelist)+1)//4]\n",
    "    interval=2*(Q3-Q1)/(len(orgdifferencelist)**(1/3))\n",
    "    maxInterval=math.ceil((Q3+(Q3-Q1)*1.5)/interval)\n",
    "    def gen_new_pattern(pattern_dict,index_list,father_pattern,database,min_sup,min_interval):\n",
    "        # father_pattern 字符串格式的字符数组\n",
    "        # father_pattern=[int(num) for num in father_pattern[1:-1].split(\",\")]\n",
    "        # father_pattern = father_pattern[1:-1].split(\",\")\n",
    "        \n",
    "        length=len(father_pattern)\n",
    "        temp_dict={}\n",
    "        for pair in index_list:\n",
    "            trace_index=pair[0]\n",
    "            pattern_index=pair[1]\n",
    "            mother_trace=database[trace_index]\n",
    "            for i in range(1,min_interval+1):\n",
    "                event_index=pattern_index+i\n",
    "                if event_index>=len(mother_trace):\n",
    "                    continue\n",
    "                act=mother_trace[event_index]\n",
    "                if act not in temp_dict:\n",
    "                    temp_dict[act]=[[trace_index,event_index]]\n",
    "                else:\n",
    "                    temp_dict[act].append([trace_index,event_index])\n",
    "        for k,v in temp_dict.items():\n",
    "            if len(v)>=min_sup:\n",
    "                new_pattern=father_pattern+\",\"+k\n",
    "                pattern_dict[new_pattern]=v\n",
    "                gen_new_pattern(pattern_dict,v,new_pattern,database,min_sup,min_interval)\n",
    "            \n",
    "    def gen_one_dict(database,min_sup):\n",
    "        tp_dict={}\n",
    "        for i,trace in enumerate(database):\n",
    "            for j,item in enumerate(trace):\n",
    "                # tp_dict[str([item])]=tp_dict.get(str([item]),[]).append([i,j])\n",
    "                if item not in tp_dict:\n",
    "                    tp_dict[item]=[[i,j]]\n",
    "                else:\n",
    "                    tp_dict[item].append([i,j])\n",
    "        count_dict={}\n",
    "\n",
    "        for key,v in tp_dict.items():\n",
    "            if len(v)>=min_sup:\n",
    "                count_dict[key]=v\n",
    "\n",
    "        return count_dict\n",
    "            \n",
    "    def res_clustering():\n",
    "        res_act_list=[[0 for j in range(len(all_events))] for i in range(len(all_resource))]\n",
    "        abnormal_res_act_list=[0 for j in range(len(all_events))]\n",
    "        for act_trace,res_trace in zip(act,res):\n",
    "            for activity,resource in zip(act_trace,res_trace):\n",
    "                if resource in normal_res:\n",
    "                    res_act_list[resource][activity]+=1\n",
    "                else:\n",
    "                    abnormal_res_act_list[activity]+=1\n",
    "        for abn_res in abnormal_res:\n",
    "            res_act_list[abn_res]=copy.deepcopy(abnormal_res_act_list)\n",
    "            \n",
    "        for i in range(len(res_act_list)):\n",
    "            tsum=sum(res_act_list[i])\n",
    "            for j in range(len(res_act_list[i])):\n",
    "                if tsum!=0:\n",
    "                    res_act_list[i][j]=(res_act_list[i][j])/(tsum)\n",
    "        res_act_array=np.array(res_act_list)\n",
    "        def cosine_similarity(a,b):\n",
    "            return a.dot(b)/(np.linalg.norm(a)*np.linalg.norm(b))\n",
    "        Similarity_matrix=[[0 for j in range(len(all_resource))] for i in range(len(all_resource))]\n",
    "        for i in range(len(all_resource)):\n",
    "            for j in range(len(all_resource)):\n",
    "                Similarity_matrix[i][j]=cosine_similarity(res_act_array[i],res_act_array[j])\n",
    "        Similarity_matrix\n",
    "        from sklearn.cluster import SpectralClustering\n",
    "        clustering = SpectralClustering().fit_predict(Similarity_matrix)\n",
    "        return clustering\n",
    "    res_cluster_label=res_clustering()\n",
    "    res_cluster_log=[[res_cluster_label[resource] for resource in res_trace] for res_trace in res]\n",
    "    def Discrete_continuous_properties(act,properties,interval_size,max_interval):\n",
    "        act_prop_data=[]\n",
    "        for act_trace,con_trace in zip(act,properties):\n",
    "            temp_trace=[]\n",
    "            for activity,amount in zip(act_trace,con_trace):\n",
    "                dis_amount=str(math.floor(amount/interval_size) if (amount/interval_size) < max_interval else max_interval)\n",
    "                item=str(activity)+\";\"+dis_amount\n",
    "                temp_trace.append(item)\n",
    "            act_prop_data.append(temp_trace)\n",
    "        return act_prop_data\n",
    "    def Bound_discrete_properties(act,properties):\n",
    "        act_prop_data=[]\n",
    "        for act_trace,dis_trace in zip(act,properties):\n",
    "            temp_trace=[]\n",
    "            for activity,item in zip(act_trace,dis_trace):\n",
    "                pair=str(activity)+\";\"+str(item)\n",
    "                temp_trace.append(pair)\n",
    "            act_prop_data.append(temp_trace)\n",
    "        return act_prop_data\n",
    "    def generate_act_pattern(min_sup):\n",
    "        act_data=[]\n",
    "        for trace in act:\n",
    "            act_data.append([str(act) for act in trace])\n",
    "        pattern_dict_one=gen_one_dict(act_data,min_sup)\n",
    "        min_interval=1\n",
    "        pattern_dict={}\n",
    "        for key,value in pattern_dict_one.items():\n",
    "            gen_new_pattern(pattern_dict,value,key,act_data,min_sup,min_interval)\n",
    "        return pattern_dict\n",
    "    def generate_act_prop_pattern(act,props,min_sup,mode,interval_size,max_interval):\n",
    "        if mode:\n",
    "            act_prop_data=Discrete_continuous_properties(act,props,interval_size,max_interval)\n",
    "        else:\n",
    "            act_prop_data=Bound_discrete_properties(act,props)\n",
    "        # print(\"props bound done\")\n",
    "        pattern_act_props_one=gen_one_dict(act_prop_data,min_sup)\n",
    "        # print(\"one dict generatered\")\n",
    "        min_interval=1\n",
    "        pattern_dict={}\n",
    "        for key,value in pattern_act_props_one.items():\n",
    "            gen_new_pattern(pattern_dict,value,key,act_prop_data,min_sup,min_interval)\n",
    "        return pattern_dict\n",
    "\n",
    "    def pattern_mining(alpha):\n",
    "        min_sup=len(act)*alpha\n",
    "        start=time.time()\n",
    "        pattern_act_dict=generate_act_pattern(min_sup)\n",
    "        pattern_act_time_dict=generate_act_prop_pattern(act,time_difference_lists,min_sup,1,interval,maxInterval)\n",
    "        pattern_act_res_dict=generate_act_prop_pattern(act,res_cluster_log,min_sup,0,0,0)\n",
    "        end=time.time()\n",
    "        pickle.dump(pattern_act_dict,open(\"datasets_results/\"+namedataset+\"/\"+\"fold\"+str(step)+\"/pattern_act_dict_\"+str(alpha)[:3]+\".pkl\",\"wb\"))\n",
    "        pickle.dump(pattern_act_time_dict,open(\"datasets_results/\"+namedataset+\"/\"+\"fold\"+str(step)+\"/pattern_act_time_dict_\"+str(alpha)[:3]+\".pkl\",\"wb\"))\n",
    "        pickle.dump(pattern_act_res_dict,open(\"datasets_results/\"+namedataset+\"/\"+\"fold\"+str(step)+\"/pattern_act_res_dict_\"+str(alpha)[:3]+\".pkl\",\"wb\"))\n",
    "        pickle.dump(res_cluster_label,open(\"datasets_results/\"+namedataset+\"/\"+\"fold\"+str(step)+\"/res_cluster_label_\"+str(alpha)[:3]+\".pkl\",\"wb\"))\n",
    "        pickle.dump(maxInterval,open(\"datasets_results/\"+namedataset+\"/\"+\"fold\"+str(step)+\"/maxInterval_\"+str(alpha)[:3]+\".pkl\",\"wb\"))\n",
    "        \n",
    "        # print(\"generate time \",alpha,\"  \",end-start)\n",
    "        # print(\"act pattern length \",len(pattern_act_dict))\n",
    "        # print(\"act res pattern length \",len(pattern_act_res_dict))\n",
    "        # print(\"act time pattern length \",len(pattern_act_time_dict))\n",
    "        return end-start,len(pattern_act_dict),len(pattern_act_res_dict),len(pattern_act_time_dict)\n",
    "\n",
    "    with open(\"datasets_results/\"+namedataset+\"/fold\"+str(step)+\"/rq4.csv\",\"w\") as f:\n",
    "        writer=csv.writer(f)\n",
    "        writer.writerow([\"\",0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])\n",
    "        times=[]\n",
    "        l1s=[]\n",
    "        l2s=[]\n",
    "        l3s=[]\n",
    "        for i in range(1,11):\n",
    "            alpha=0.1*i\n",
    "            time1,l1,l2,l3=pattern_mining(alpha)\n",
    "            times.append(time1)\n",
    "            l1s.append(l1)\n",
    "            l2s.append(l2)\n",
    "            l3s.append(l3)\n",
    "        row=[\"time\"]+times\n",
    "        writer.writerow(row)\n",
    "        row=[\"pattern1\"]+l1s\n",
    "        writer.writerow(row)\n",
    "        row=[\"pattern2\"]+l2s\n",
    "        writer.writerow(row)\n",
    "        row=[\"pattern3\"]+l3s\n",
    "        writer.writerow(row)\n",
    "\n",
    "        \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('py3.9')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2e67272651055d1832734db27dbe5c78ae7e6195044c32c87375667fa374755b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}