From e3a3fa0bbb94197b6d30a561c5f6c4966253b666 Mon Sep 17 00:00:00 2001 From: Weixiang Yu Date: Tue, 12 Sep 2017 20:45:00 -0400 Subject: [PATCH 1/3] rm unneccesary index creation --- vizic/connection.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vizic/connection.py b/vizic/connection.py index 9e521bc..350aec9 100644 --- a/vizic/connection.py +++ b/vizic/connection.py @@ -16,7 +16,6 @@ def __init__(self): self._des_crs = [] self.x_range = 0 self.y_range = 0 - self.db_maxZoom = 8 self._minMax = {} self.cat_ct = 1 @@ -183,7 +182,7 @@ def add_to_old(self, df, coll_name, map_dict=None): if map_dict is not None: for k in map_dict.keys(): df[k] = df[map_dict[k]] - print(list(df.columns)) + clms = [x.upper() for x in list(df.columns)] if not set(['RA', 'DEC']).issubset(set(clms)): raise Exception("RA, DEC is required for visualization!") @@ -193,7 +192,7 @@ def add_to_old(self, df, coll_name, map_dict=None): elif ('RADIUS' not in clms and not set(['A_IMAGE', 'B_IMAGE', 'THETA_IMAGE']).issubset(set(clms))) or \ db_meta.point: - print(db_meta.point) + coll.point = True print('Objects as points, slow performance') else: @@ -334,5 +333,7 @@ def _insert_data(self, df, coll): collection = self.db[coll.name] collection.insert_many(data_d, ordered=False) collection.update_one({'_id': 'meta'}, {'$set':{'adjust': coll._des_crs, 'xRange': coll.x_range, 'yRange': coll.y_range, 'minmax': coll._minMax, 'radius':coll.radius,'point':coll.point, 'catCt':coll.cat_ct}}, upsert=True) - collection.create_index([('loc', pmg.GEO2D)], name='geo_loc_2d', min=-90, max=360) - collection.create_index([('b', pmg.ASCENDING)], name='semi_axis') + + if coll.cat_ct == 1: + collection.create_index([('loc', pmg.GEO2D)], name='geo_loc_2d', min=-90, max=360) + collection.create_index([('b', pmg.ASCENDING)], name='semi_axis') From 4f9aaa37a851f0b388b7bbe13801c400314bd43c Mon Sep 17 00:00:00 2001 From: Weixiang Yu Date: Tue, 12 Sep 2017 20:54:07 -0400 Subject: [PATCH 2/3] changed function names --- vizic/astroleaflet.py | 1 + vizic/connection.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vizic/astroleaflet.py b/vizic/astroleaflet.py index 4349403..510b488 100644 --- a/vizic/astroleaflet.py +++ b/vizic/astroleaflet.py @@ -10,6 +10,7 @@ import requests from notebook.utils import url_path_join from .utils import cut_tree, get_mst, get_m_index, get_vert_bbox +from .connection import collection class AstroMap(Map): diff --git a/vizic/connection.py b/vizic/connection.py index 350aec9..6d46b8e 100644 --- a/vizic/connection.py +++ b/vizic/connection.py @@ -116,7 +116,7 @@ def show_circles(self): return circles - def import_new(self, df, coll_name, map_dict=None): + def to_new(self, df, coll_name, map_dict=None): """Import new catalog without creating a map layer. Args: @@ -128,7 +128,7 @@ def import_new(self, df, coll_name, map_dict=None): exist_colls = self.show_catalogs() if coll_name in exist_colls: - raise Exception('Provided collection name already exists, use a different name or use function import_exists().') + raise Exception('Provided collection name already exists, use a different name or use function to_exists().') coll = Collection() coll.name = coll_name @@ -162,7 +162,7 @@ def import_new(self, df, coll_name, map_dict=None): self._insert_data(df_r, coll) - def add_to_old(self, df, coll_name, map_dict=None): + def to_exists(self, df, coll_name, map_dict=None): """Add new data to existing catalog collection. Args: @@ -175,7 +175,7 @@ def add_to_old(self, df, coll_name, map_dict=None): if coll_name in exist_colls: db_meta = self.read_meta(coll_name) else: - raise Exception('Provided collection name does not exist, please use function import_new()') + raise Exception('Provided collection name does not exist, please use function to_new()') coll = Collection() coll.name = coll_name From e1facc74412fee28719bec3fa27afb9863fdb9c1 Mon Sep 17 00:00:00 2001 From: Weixiang Yu Date: Tue, 12 Sep 2017 22:21:03 -0400 Subject: [PATCH 3/3] move all data-related to connection.py & add update_mata() --- vizic/astroleaflet.py | 132 ++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 83 deletions(-) diff --git a/vizic/astroleaflet.py b/vizic/astroleaflet.py index 510b488..42a3db4 100644 --- a/vizic/astroleaflet.py +++ b/vizic/astroleaflet.py @@ -10,7 +10,7 @@ import requests from notebook.utils import url_path_join from .utils import cut_tree, get_mst, get_m_index, get_vert_bbox -from .connection import collection +from .connection import Collection class AstroMap(Map): @@ -264,17 +264,16 @@ def _checkInput(self, coll_name, map_dict): coll_name: MongoDB collection name for the new catalog. map_dict(dict): Dataframe columns mapping dictionary. """ + + coll = Collection() + exist_colls = self.db.collection_names() + if not self.collection == '': - meta = self.db[self.collection].find_one({'_id': 'meta'}) - self._des_crs = meta['adjust'] - self.x_range = meta['xRange'] - self.y_range = meta['yRange'] - self.__minMax = meta['minmax'] - (self.radius, self.point) = (meta['radius'], meta['point']) - self.cat_ct = meta['catCt'] + if self.collection not in exist_colls: + raise Exception('Provided collection does not exist!') + coll = self.connection.read_meta(self.collection) elif self.df is not None: clms = [x.upper() for x in list(self.df.columns)] - exist_colls = self.db.collection_names() if not set(['RA', 'DEC']).issubset(set(clms)): raise Exception("RA, DEC is required for visualization!") @@ -282,16 +281,16 @@ def _checkInput(self, coll_name, map_dict): print('No shape information provided') if 'RADIUS' in clms: print('Will use radius for filtering!') - self.radius = True + coll.radius = True else: print('Object as point, slow performance!') - self.point = True + coll.point = True if coll_name is not None and coll_name in exist_colls: raise Exception('Collectoin name already exists, try to use a different name or use existing collection.') - df_r, self._des_crs = self._data_prep(self.df) - self.x_range = self._des_crs[2]*256 - self.y_range = self._des_crs[3]*256 + df_r, coll._des_crs = self.connection._data_prep(self.df, coll) + coll.x_range = coll._des_crs[2]*256 + coll.y_range = coll._des_crs[3]*256 # drop created mapped columns before ingecting data if map_dict is not None: @@ -299,78 +298,31 @@ def _checkInput(self, coll_name, map_dict): if x.upper() not in ['RA', 'DEC']] df_r.drop(col_keys,axis=1, inplace=True) for k in col_keys: - self.__minMax.pop(k, None) + coll._minMax.pop(k, None) - self._insert_data(df_r, coll_name) - else: - raise Exception('Need to provide a collection name or a pandas dataframe!') + # check if name given, if not using uuid + if coll_name is not None: + coll.name = coll_name + else: + coll_id = str(uuid.uuid4()) + coll.name = coll_id - def _data_prep(self, df): - """Private method for formatting catalog. - - Metadata for catalog provided in a pandas dataframe is extracted here. - Corresponding tile ID for each object in the catalog is caculated and - inserted into the dataframe, so as the mapped coordinates and - shapes/sizes for the objects. - - Args: - df: A pandas dataframe containning the catalog. - - Returns: - A new pandas dataframe with added columns and a list specifying - the coordinate scale. - - """ - dff = df.copy() - dff.columns = [x.upper() for x in dff.columns] - (xMax, xMin) = (dff['RA'].max(), dff['RA'].min()) - (yMax, yMin) = (dff['DEC'].max(), dff['DEC'].min()) - x_range = xMax - xMin - y_range = yMax - yMin - - # find field min and max - for col in dff.columns: - if dff[col].dtype.kind == 'f': - self.__minMax[col] = [dff[col].min(), dff[col].max()] - - if self.radius: - dff.loc[:, 'b'] = dff.loc[:, 'RADIUS'].apply(lambda x: x*0.267/3600) - dff['theta'] = 0 - elif self.point: - dff['b'] = 360 - dff['theta'] = 0 + self.connection._insert_data(df_r, coll) else: - dff.loc[:, 'a'] = dff.loc[:, 'A_IMAGE'].apply(lambda x: x*0.267/3600) - dff.loc[:, 'b'] = dff.loc[:, 'B_IMAGE'].apply(lambda x: x*0.267/3600) - dff.loc[:, 'theta'] = dff.loc[:, 'THETA_IMAGE'] - - # assign 'loc' columns for geoIndex in Mongo - dff['loc'] = list(zip(dff.RA, dff.DEC)) - - xScale = x_range/256 - yScale = y_range/256 - return dff, [xMin, yMax, xScale, yScale] + raise Exception('Need to provide a collection name or a pandas dataframe!') - def _insert_data(self, df, coll_name): - """Private method to insert a catalog into database. + # now assign collection object values to GridLayer + self._des_crs = coll._des_crs + self.x_range = coll.x_range + self.y_range = coll.y_range + self.point = coll.point + self.radius = coll.radius + self.__minMax = coll._minMax + self.cat_ct = coll.cat_ct + self.collection = coll.name - Args: - df: A pandas dataframe with correctly formatted catalog. - coll_name: MongoDB collection name for the new catalog. - """ - if coll_name is not None: - self.collection = coll_name - if self.collection == '': - coll_id = str(uuid.uuid4()) - self.collection = coll_id - - df['cat_rank'] = 1 - data_d = df.to_dict(orient='records') - coll = self.db[self.collection] - coll.insert_many(data_d, ordered=False) - coll.insert_one({'_id': 'meta', 'adjust': self._des_crs, 'xRange': self.x_range, 'yRange': self.y_range, 'minmax': self.__minMax, 'radius': self.radius,'point': self.point, 'catCt': 1}) - coll.create_index([('loc', pmg.GEO2D)], name='geo_loc_2d', min=-90, max=360) - coll.create_index([('b', pmg.ASCENDING)], name='semi_axis') + # The center is required, don't remove + self.center = [self._des_crs[1]-self.y_range/2, self._des_crs[0]+self.x_range/2] def push_data(self, url): """Update server extension with newly displayed catalog. @@ -381,8 +333,7 @@ def push_data(self, url): Args: url(str): The Jupyter server address. """ - # The center is required, don't remove - self.center = [self._des_crs[1]-self.y_range/2, self._des_crs[0]+self.x_range/2] + mRange = (self.x_range + self.y_range)/2 body = { 'collection': self.collection, @@ -392,6 +343,21 @@ def push_data(self, url): push_url = url_path_join(url, '/rangeinfo/') req = requests.post(push_url, data=body) + def update_meta(self): + """Update meta information after new data added.""" + + coll = self.connection.read_meta(self.collection) + self._des_crs = coll._des_crs + self.x_range = coll.x_range + self.y_range = coll.y_range + self.__minMax = coll._minMax + self.cat_ct = coll.cat_ct + # update info at Jupyter server + self.push_data(self._server_url) + # update the AstroMap meta + self._map.center = self.center = [self._des_crs[1]-self.y_range/2, self._des_crs[0]+self.x_range/2] + self._map._des_crs = self._des_crs + def _handle_leaflet_event(self, _, content, buffers): """Handle leaflet events trigged.""" if content.get('event', '') == 'popup: click':