Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cleanup and consistency improvement for Multiple catalog #17

Merged
merged 3 commits into from
Sep 13, 2017
Merged

cleanup and consistency improvement for Multiple catalog #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e3a3fa0
rm unneccesary index creation
Sep 13, 2017
4f9aaa3
changed function names
Sep 13, 2017
e1facc7
move all data-related to connection.py & add update_mata()
Sep 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 49 additions & 82 deletions vizic/astroleaflet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import requests
from notebook.utils import url_path_join
from .utils import cut_tree, get_mst, get_m_index, get_vert_bbox
from .connection import Collection


class AstroMap(Map):
Expand Down Expand Up @@ -263,113 +264,65 @@ def _checkInput(self, coll_name, map_dict):
coll_name: MongoDB collection name for the new catalog.
map_dict(dict): Dataframe columns mapping dictionary.
"""

coll = Collection()
exist_colls = self.db.collection_names()

if not self.collection == '':
meta = self.db[self.collection].find_one({'_id': 'meta'})
self._des_crs = meta['adjust']
self.x_range = meta['xRange']
self.y_range = meta['yRange']
self.__minMax = meta['minmax']
(self.radius, self.point) = (meta['radius'], meta['point'])
self.cat_ct = meta['catCt']
if self.collection not in exist_colls:
raise Exception('Provided collection does not exist!')
coll = self.connection.read_meta(self.collection)
elif self.df is not None:
clms = [x.upper() for x in list(self.df.columns)]
exist_colls = self.db.collection_names()

if not set(['RA', 'DEC']).issubset(set(clms)):
raise Exception("RA, DEC is required for visualization!")
if not set(['A_IMAGE', 'B_IMAGE', 'THETA_IMAGE']).issubset(set(clms)):
print('No shape information provided')
if 'RADIUS' in clms:
print('Will use radius for filtering!')
self.radius = True
coll.radius = True
else:
print('Object as point, slow performance!')
self.point = True
coll.point = True

if coll_name is not None and coll_name in exist_colls:
raise Exception('Collectoin name already exists, try to use a different name or use existing collection.')
df_r, self._des_crs = self._data_prep(self.df)
self.x_range = self._des_crs[2]*256
self.y_range = self._des_crs[3]*256
df_r, coll._des_crs = self.connection._data_prep(self.df, coll)
coll.x_range = coll._des_crs[2]*256
coll.y_range = coll._des_crs[3]*256

# drop created mapped columns before ingecting data
if map_dict is not None:
col_keys = [x.upper() for x in map_dict.keys()
if x.upper() not in ['RA', 'DEC']]
df_r.drop(col_keys,axis=1, inplace=True)
for k in col_keys:
self.__minMax.pop(k, None)
coll._minMax.pop(k, None)

self._insert_data(df_r, coll_name)
else:
raise Exception('Need to provide a collection name or a pandas dataframe!')
# check if name given, if not using uuid
if coll_name is not None:
coll.name = coll_name
else:
coll_id = str(uuid.uuid4())
coll.name = coll_id

def _data_prep(self, df):
"""Private method for formatting catalog.

Metadata for catalog provided in a pandas dataframe is extracted here.
Corresponding tile ID for each object in the catalog is caculated and
inserted into the dataframe, so as the mapped coordinates and
shapes/sizes for the objects.

Args:
df: A pandas dataframe containning the catalog.

Returns:
A new pandas dataframe with added columns and a list specifying
the coordinate scale.

"""
dff = df.copy()
dff.columns = [x.upper() for x in dff.columns]
(xMax, xMin) = (dff['RA'].max(), dff['RA'].min())
(yMax, yMin) = (dff['DEC'].max(), dff['DEC'].min())
x_range = xMax - xMin
y_range = yMax - yMin

# find field min and max
for col in dff.columns:
if dff[col].dtype.kind == 'f':
self.__minMax[col] = [dff[col].min(), dff[col].max()]

if self.radius:
dff.loc[:, 'b'] = dff.loc[:, 'RADIUS'].apply(lambda x: x*0.267/3600)
dff['theta'] = 0
elif self.point:
dff['b'] = 360
dff['theta'] = 0
self.connection._insert_data(df_r, coll)
else:
dff.loc[:, 'a'] = dff.loc[:, 'A_IMAGE'].apply(lambda x: x*0.267/3600)
dff.loc[:, 'b'] = dff.loc[:, 'B_IMAGE'].apply(lambda x: x*0.267/3600)
dff.loc[:, 'theta'] = dff.loc[:, 'THETA_IMAGE']

# assign 'loc' columns for geoIndex in Mongo
dff['loc'] = list(zip(dff.RA, dff.DEC))

xScale = x_range/256
yScale = y_range/256
return dff, [xMin, yMax, xScale, yScale]
raise Exception('Need to provide a collection name or a pandas dataframe!')

def _insert_data(self, df, coll_name):
"""Private method to insert a catalog into database.
# now assign collection object values to GridLayer
self._des_crs = coll._des_crs
self.x_range = coll.x_range
self.y_range = coll.y_range
self.point = coll.point
self.radius = coll.radius
self.__minMax = coll._minMax
self.cat_ct = coll.cat_ct
self.collection = coll.name

Args:
df: A pandas dataframe with correctly formatted catalog.
coll_name: MongoDB collection name for the new catalog.
"""
if coll_name is not None:
self.collection = coll_name
if self.collection == '':
coll_id = str(uuid.uuid4())
self.collection = coll_id

df['cat_rank'] = 1
data_d = df.to_dict(orient='records')
coll = self.db[self.collection]
coll.insert_many(data_d, ordered=False)
coll.insert_one({'_id': 'meta', 'adjust': self._des_crs, 'xRange': self.x_range, 'yRange': self.y_range, 'minmax': self.__minMax, 'radius': self.radius,'point': self.point, 'catCt': 1})
coll.create_index([('loc', pmg.GEO2D)], name='geo_loc_2d', min=-90, max=360)
coll.create_index([('b', pmg.ASCENDING)], name='semi_axis')
# The center is required, don't remove
self.center = [self._des_crs[1]-self.y_range/2, self._des_crs[0]+self.x_range/2]

def push_data(self, url):
"""Update server extension with newly displayed catalog.
Expand All @@ -380,8 +333,7 @@ def push_data(self, url):
Args:
url(str): The Jupyter server address.
"""
# The center is required, don't remove
self.center = [self._des_crs[1]-self.y_range/2, self._des_crs[0]+self.x_range/2]

mRange = (self.x_range + self.y_range)/2
body = {
'collection': self.collection,
Expand All @@ -391,6 +343,21 @@ def push_data(self, url):
push_url = url_path_join(url, '/rangeinfo/')
req = requests.post(push_url, data=body)

def update_meta(self):
"""Update meta information after new data added."""

coll = self.connection.read_meta(self.collection)
self._des_crs = coll._des_crs
self.x_range = coll.x_range
self.y_range = coll.y_range
self.__minMax = coll._minMax
self.cat_ct = coll.cat_ct
# update info at Jupyter server
self.push_data(self._server_url)
# update the AstroMap meta
self._map.center = self.center = [self._des_crs[1]-self.y_range/2, self._des_crs[0]+self.x_range/2]
self._map._des_crs = self._des_crs

def _handle_leaflet_event(self, _, content, buffers):
"""Handle leaflet events trigged."""
if content.get('event', '') == 'popup: click':
Expand Down
19 changes: 10 additions & 9 deletions vizic/connection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ def __init__(self):
self._des_crs = []
self.x_range = 0
self.y_range = 0
self.db_maxZoom = 8
self._minMax = {}
self.cat_ct = 1

Expand Down Expand Up @@ -117,7 +116,7 @@ def show_circles(self):

return circles

def import_new(self, df, coll_name, map_dict=None):
def to_new(self, df, coll_name, map_dict=None):
"""Import new catalog without creating a map layer.

Args:
Expand All @@ -129,7 +128,7 @@ def import_new(self, df, coll_name, map_dict=None):

exist_colls = self.show_catalogs()
if coll_name in exist_colls:
raise Exception('Provided collection name already exists, use a different name or use function import_exists().')
raise Exception('Provided collection name already exists, use a different name or use function to_exists().')

coll = Collection()
coll.name = coll_name
Expand Down Expand Up @@ -163,7 +162,7 @@ def import_new(self, df, coll_name, map_dict=None):

self._insert_data(df_r, coll)

def add_to_old(self, df, coll_name, map_dict=None):
def to_exists(self, df, coll_name, map_dict=None):
"""Add new data to existing catalog collection.

Args:
Expand All @@ -176,14 +175,14 @@ def add_to_old(self, df, coll_name, map_dict=None):
if coll_name in exist_colls:
db_meta = self.read_meta(coll_name)
else:
raise Exception('Provided collection name does not exist, please use function import_new()')
raise Exception('Provided collection name does not exist, please use function to_new()')

coll = Collection()
coll.name = coll_name
if map_dict is not None:
for k in map_dict.keys():
df[k] = df[map_dict[k]]
print(list(df.columns))

clms = [x.upper() for x in list(df.columns)]
if not set(['RA', 'DEC']).issubset(set(clms)):
raise Exception("RA, DEC is required for visualization!")
Expand All @@ -193,7 +192,7 @@ def add_to_old(self, df, coll_name, map_dict=None):
elif ('RADIUS' not in clms and
not set(['A_IMAGE', 'B_IMAGE', 'THETA_IMAGE']).issubset(set(clms))) or \
db_meta.point:
print(db_meta.point)

coll.point = True
print('Objects as points, slow performance')
else:
Expand Down Expand Up @@ -334,5 +333,7 @@ def _insert_data(self, df, coll):
collection = self.db[coll.name]
collection.insert_many(data_d, ordered=False)
collection.update_one({'_id': 'meta'}, {'$set':{'adjust': coll._des_crs, 'xRange': coll.x_range, 'yRange': coll.y_range, 'minmax': coll._minMax, 'radius':coll.radius,'point':coll.point, 'catCt':coll.cat_ct}}, upsert=True)
collection.create_index([('loc', pmg.GEO2D)], name='geo_loc_2d', min=-90, max=360)
collection.create_index([('b', pmg.ASCENDING)], name='semi_axis')

if coll.cat_ct == 1:
collection.create_index([('loc', pmg.GEO2D)], name='geo_loc_2d', min=-90, max=360)
collection.create_index([('b', pmg.ASCENDING)], name='semi_axis')