cleanup

main.py

@@ -13,9 +13,13 @@ import sqlite3
 
 conn = sqlite3.connect("rpd.db")
 cursor = conn.cursor()
+
+# uncomment on a first run
+
 #cursor.execute('DROP TABLE IF EXISTS courses;')
 #cursor.execute('CREATE TABLE courses (chapter TEXT NOT NULL, title TEXT NOT NULL, year INTEGER NOT NULL, total INTEGER NOT NULL, semesters TEXT NOT NULL, complexity INTEGER NOT NULL, competents TEXT NOT NULL, control TEXT NOT NULL);')
 #cursor.execute('CREATE TABLE courses_clean (chapter TEXT NOT NULL, title TEXT NOT NULL, year INTEGER NOT NULL, total INTEGER NOT NULL, semesters TEXT NOT NULL, complexity INTEGER NOT NULL, competents TEXT NOT NULL, control TEXT NOT NULL);')
+#cursor.execute('CREATE TABLE courses_clean (chapter TEXT NOT NULL, title TEXT NOT NULL, year INTEGER NOT NULL, total INTEGER NOT NULL, semesters TEXT NOT NULL, complexity INTEGER NOT NULL, competents TEXT NOT NULL, control TEXT NOT NULL);')
 #conn.commit()
 
 
@@ -133,7 +137,7 @@ def dash(s):
 
 def read_short_competents_description():
     competents_short = {}
-    with open('/Users/jim/Nextcloud/Documents/РПД/source data/Компетенции_кратко.csv', newline='') as csvfile:
+    with open('~/Nextcloud/Documents/РПД/source data/Компетенции_кратко.csv', newline='') as csvfile:
         comp_reader = csv.reader(csvfile, delimiter=';', quotechar="\"")
         for row in comp_reader:
             competents_short[row[0].strip()] = row[1].strip()
@@ -143,7 +147,7 @@ def read_short_competents_description():
 
 def read_long_competents_description():
     competents = {}
-    with open('/Users/jim/Nextcloud/Documents/РПД/source data/Компетенции.csv', newline='') as csvfile:
+    with open('~/Nextcloud/Documents/РПД/source data/Компетенции.csv', newline='') as csvfile:
         comp_reader = csv.reader(csvfile, delimiter=';', quotechar="\"")
         next(comp_reader, None)
         for row in comp_reader:
@@ -197,7 +201,7 @@ def fill_competents(filename, rows):
     for row in up_sheet.iter_rows(min_row=3, max_row=rows, values_only=True):
         title = row[1].replace('\n', ' ').strip()
         competents = parse_competents(row[90].replace("\n", "").replace(' ', ''))
-        #print(title, competents)
+
         q = 'update courses set competents=? where title=?;'
         cursor.execute(q, (json.dumps(competents), str(title)))
         conn.commit()
@@ -210,12 +214,12 @@ def read_course(filename, rows, read_compenents, year, chapter):
 
     for row in up_sheet.iter_rows(min_row=3, max_row=rows, values_only=True):
         title = row[1].replace('\n', ' ').strip()
-        #print(title)
+
         control = fill_control(row)
 
         total = xint(row[8])
         total_complexity = xint(row[13])
-        # print(title, total, total_complexity)
+
         first_semester_start = 14
         second_semester_start = 22
         third_semester_start = 30
@@ -384,6 +388,13 @@ def read_course(filename, rows, read_compenents, year, chapter):
             year=year,
             courses=courses)
 
+
+def fill_clean_courses():
+    q = 'insert into courses_clean select * from courses where not competents="[]" group by title, year, total, semesters, complexity, competents, control;'
+    cursor.execute(q)
+    conn.commit()
+
+
 # # 2017
 #
 # mkk_2017 = read_course(filename='/Users/jim/Nextcloud/Documents/РПД/Новые учебные планы (xls)/МКК2017.xlsx', rows=68, read_compenents=False, year=2017, chapter='МКК')
@@ -415,6 +426,8 @@ def read_course(filename, rows, read_compenents, year, chapter):
 # fill_competents(filename='/Users/jim/Nextcloud/Documents/РПД/Старые учебные планы (xls)/2017/Преподы2017 (факультативы).xlsx', rows=59)
 # fill_competents(filename='/Users/jim/Nextcloud/Documents/РПД/Старые учебные планы (xls)/2017/Перевод2017 (факультативы).xlsx', rows=71)
 #
+# fill_clean_courses()
+#
 
 
 def semesters_to_dict(semesters):
@@ -454,8 +467,6 @@ def generate_reports():
     competents_short = read_short_competents_description()
     competents_long = read_long_competents_description()
 
-    mapping = read_mapping()
-
     q = 'select distinct title from courses_clean'
     cursor.execute(q)
     course_titles = cursor.fetchall()