Scripting access of django databases

app/management/commands/addorganism.py

@@ -0,0 +1,29 @@
+from app.models import Organism
+from django.core.management.base import BaseCommand
+import django
+from add_func import display_name, short_name, get_description, get_taxid
+
+id_baseurl = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=taxonomy&retmode=json&term='
+wiki_url1 = 'https://en.wikipedia.org/w/api.php?action=query&list=search&srprop=snippet&srlimit=1&format=json&srsearch='
+wiki_url2 = 'https://en.wikipedia.org/w/api.php?action=query&prop=extracts&format=json&exintro=true&titles='
+
+class Command(BaseCommand):
+
+    def add_arguments(self,parser):
+        parser.add_argument('Genus_Species',nargs='+',type=str)
+        #parser.add_argument('Species',nargs='*',type=str)
+        #parser.add_argument('Species2',nargs='?',type=str)
+    def handle(self,*args,**options):
+
+        name = display_name(options)
+        shortname = short_name(name)
+        url1 = wiki_url1 + name
+        description = get_description(url1,wiki_url2)
+        tax_id = get_taxid(id_baseurl,name)
+        new_org = Organism(display_name=name, short_name=shortname, description=description, tax_id=tax_id)
+
+        try:
+            new_org.save()
+            print("Succeessfully add to database")
+        except django.db.utils.IntegrityError:
+            print("adding database failed, check if this organism is already in the database and try again")

blast/management/commands/addblast.py

@@ -0,0 +1,32 @@
+from blast.models import BlastDb
+from django.core.management.base import BaseCommand
+#from app.models import Organism
+#sys.path.append('genomics-workspace/app/management/commands/add_func.py')
+from add_func import get_organism, display_name, get_path, get_type, get_molecule, get_dataset
+
+class Command(BaseCommand):
+
+    def add_arguments(self,parser):
+        parser.add_argument('Genus_Species',nargs='+',type=str)
+        parser.add_argument('-t','--type',nargs='+',type=str,help='please enter nucleotide or peptide and enter Genome Assembly or Protein or Transcript')
+        parser.add_argument('-f','--filename',nargs=1,type=str)
+
+    def handle(self,*args,**options):
+
+        name=display_name(options)
+        organism = get_organism(name)
+        if organism:#check whether organism is exist or not
+            molecule2,molecule_str = get_molecule(options)
+            dataset,dataset_str = get_dataset(options)
+            blast_type = get_type(dataset,molecule2,molecule_str,dataset_str)
+            title = options['filename'][0]
+            fasta_file_path = get_path('blast',title)
+            new_db = BlastDb(organism = organism, type = blast_type, fasta_file = fasta_file_path, title = title, description = '', is_shown = False )
+            new_db.save()
+            print("you can move to makeblastdb and populate sequence step")
+            #except django.db.utils.IntegrityError:
+                #print("This database already exists")
+                #sys.exit(0)
+        else :
+            pass
+            #TODO can use subprocess lib here to add new organism

blast/management/commands/blast_shown.py

@@ -0,0 +1,26 @@
+from blast.models import BlastDb
+from django.core.management.base import BaseCommand
+import sys
+
+class Command(BaseCommand):
+
+    def add_arguments(self,parser):
+        parser.add_argument('BlastDb', nargs='+', type=str, help='enter the blastdb name')
+        parser.add_argument('--shown', nargs='*', help= 'make blastdb show or not ex: python manage.py blast_shown [xxx.fa] [xxx.fa] --shown true/false')
+    def handle(self,*args,**options):
+
+        n=0;
+        title = options['BlastDb']
+        for title in title:
+            blast2 = BlastDb.objects.filter(title = title)
+            print options
+            n+=1
+            if options['shown'][0] == 'true':
+                blast2.update(is_shown = True)
+            elif options['shown'][0] == 'false':
+                blast2.update(is_shown = False)
+            else:
+                print("please choose  --shown for true or false")
+                sys.exit(0)
+            print("%d species finished "%n)
+        print("all done")

blast/management/commands/blast_utility.py

@@ -0,0 +1,29 @@
+from blast.models import BlastDb
+from django.core.management.base import BaseCommand
+import sys
+
+class Command(BaseCommand):
+
+    def add_arguments(self,parser):
+        parser.add_argument('BlastDb', nargs='+', type=str, help='enter the blastdb name')
+        parser.add_argument('-m','--makeblastdb', nargs='*', help = 'execute makeblastdb command to specific blastdb, ex: python manage.py blast_utility [xxx.fa] [xxx.fa] -m')
+        parser.add_argument('-p','--populatesequence', nargs='*', help = 'populate specifice blastdb, ex: python manage.py blast_utility [xxx.fa] [xxx.fa] -p')
+        #parser.add_argument('--shown', nargs='*', help= 'make blastdb show or not ex: python manage.py blast_utility [xxx.fa] [xxx.fa] --shown true/false')
+
+    def handle(self,*args,**options):
+
+        n=0;
+        title = options['BlastDb']
+        for title in title:
+            blast = BlastDb.objects.get(title = title)
+            #print blast
+            n+=1
+            if options['makeblastdb'] == []:
+                blast.makeblastdb()
+            elif options['populatesequence'] == []:
+                blast.index_fasta()
+            else:
+                print("please choose -m for makeblastd, -p for populate sequence, --shown for true or false")
+                sys.exit(0)
+            print("%d species finished "%n)
+        print("all done")

hmmer/management/commands/addhmmer.py

@@ -0,0 +1,30 @@
+from hmmer.models import HmmerDB
+from django.core.management.base import BaseCommand
+#from app.models import Organism
+#import django.db
+from add_func import get_organism, display_name, get_path
+
+class Command(BaseCommand):
+
+    def add_arguments(self,parser):
+        parser.add_argument('Genus_Species',nargs='+',type=str)
+        parser.add_argument('-f','--filename',nargs=1,type=str)
+
+    def handle(self,*args,**options):
+
+        name=display_name(options)
+        organism = get_organism(name)
+        #print options
+        if organism:#check whether organism is exist or not
+
+            title = options['filename'][0]
+            fasta_file_path = get_path('hmmer',title)
+            new_db = HmmerDB(organism = organism, fasta_file = fasta_file_path, title = title, description = '', is_shown = True )
+            new_db.save()
+            print("Success")
+            #except django.db.utils.IntegrityError:
+                #print("This database already exists")
+                #sys.exit(0)
+        else :
+            pass
+            #can use subprocess lib here to add new organism

misc/add_func.py

@@ -0,0 +1,155 @@
+from blast.models import SequenceType
+#from blast.models import BlastDb
+from app.models import Organism
+import os
+import sys
+import requests
+#from hmmer.models import HmmerDB
+
+
+def display_name(options):
+    try:
+        base_organism = options['Genus_Species'][0].lower().capitalize() + ' ' + options['Genus_Species'][1].lower()
+    except TypeError:
+        return 0
+    if len(options['Genus_Species']) == 3:
+        display_name = base_organism + ' '+ options['Genus_Species'][2].lower()
+        return display_name
+
+    else:
+        display_name = base_organism
+        return display_name
+
+def get_organism(display_name):
+
+    organism_database = Organism.objects.get(display_name = display_name)
+    if organism_database :
+        return organism_database
+    else:
+        print("check your organism name again if it still fails then check your organism database")
+        sys.exit(0)
+
+def get_path(app_name,title):
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    if app_name == 'blast':
+        path = os.path.join('blast/db',title)
+    else:
+        path = os.path.join('hmmer/db',title)
+
+    a=os.path.join(base_dir,'media',path)
+    check = os.path.isfile(a)
+    if check:
+        return path
+    else:
+        print("No fasta file in media/blast/db or media/hmmer/db")
+        sys.exit(0)
+
+def short_name(name):
+    short_name = name.split(' ')
+    short_name1 = short_name[0][0:3]
+    short_name2 = short_name[1][0:3]
+    short_name = short_name1 + short_name2
+    return short_name
+
+def get_molecule(options):
+    try:
+        molecule = options['type'][0].lower() #get molecule_type from command line
+        if molecule == 'peptide':    #change the name tp prot or nucl
+            molecule2 = 'prot'
+        elif molecule == 'nucleotide':
+            molecule2 = 'nucl'
+        else:
+            print("please enter the correct molecule_type, must be nucleotide or peptide")
+            sys.exit(0)
+    except Exception :
+        print("enter the argument complete '-t' '-f' ")
+        sys.exit(0)
+    molecule_type = SequenceType.objects.filter(molecule_type = molecule2) #get the data from molecule_type field
+    a = molecule_type[0]
+    molecule_str = a.molecule_type
+    return molecule2,molecule_str
+
+def get_dataset(options):
+
+    dataset = options['type'][1].lower().capitalize()
+    if dataset =='Genome':
+        dataset = dataset + ' ' + options['type'][2].lower().capitalize()
+    elif dataset == 'Transcript':
+        pass
+    elif dataset == 'Protein':
+        pass
+    else:
+        print('enter the correct dataset type')
+        sys.exit(0)
+    dataset_type = SequenceType.objects.filter(dataset_type = dataset)
+    b = dataset_type[0]
+    dataset_str = str(b.dataset_type)
+    return dataset,dataset_str
+
+def get_type(dataset,molecule2,molecule_str,dataset_str): #get the sequence type from SequencType Table
+
+    if molecule2 != molecule_str :
+        print("something wrong in molecule")
+    elif dataset != dataset_str :
+        print("something wrong with dataset")
+    else:
+        try:
+            dataset_type = SequenceType.objects.filter(molecule_type = molecule2, dataset_type = dataset)
+            return dataset_type[0]
+        except IndexError:
+            print("there are no {molecule} - {dataset} combination in the database".format(molecule=molecule2.capitalize(),dataset=dataset_str))
+            sys.exit(0)
+def get_description(url1,wiki_url2):
+    try:
+        re1 = requests.get(url1)
+        data1 = re1.json()
+        try:
+            title = data1['query']['search'][0]['title']
+            url2 = wiki_url2 + title
+            re2 = requests.get(url2)
+            data2 = re2.json()
+            key = data1['query']['search'][0]['pageid']
+            key = str(key)
+            description = data2['query']['pages'][key]['extract']
+            return description
+        except 	IndexError:
+            print("check your organism name again")
+            sys.exit(0)
+    except requests.exceptions.ConnectionError:
+        print("check your internet connection")
+        sys.exit(0)
+
+def get_taxid(id_baseurl,name):
+    try:
+        url = id_baseurl+ name
+        re = requests.get(url)
+        data = re.json()
+        tax_id = data['esearchresult']['idlist'][0]
+        tax_id = int(tax_id)
+        return tax_id
+    except IndexError:
+        print("make sure your name is completed and correct")
+        sys.exit(0)
+
+def delete_org(name):
+    Organism.objects.filter(display_name = name).delete()
+    return ("remove %s in database"%name)
+"""
+def delete(db, dbname):
+    tmp=[]
+    if db[0]=='all':
+        if dbname=='blast':
+            BlastDb.objects.all().delete()
+            print("remove all data in blast")
+        else:
+            HmmerDB.objects.all().delete()
+            print("remove all data in hmmer")
+    else:
+        for name in db :
+            if dbname=='blast':
+                BlastDb.objects.filter(title = name).delete()
+            else:
+                HmmerDB.objects.filter(title = name).delete()
+            tmp.append(name)
+    return "remove %s "%tmp
+"""

requirements.txt

@@ -1,4 +1,4 @@
-Django==1.11.20
+Django==1.11.22
 Markdown==2.6.6
 amqp==2.3.2
 celery==4.1.1