|
Size: 4027
Comment:
|
Size: 2264
Comment:
|
| Deletions are marked like this. | Additions are marked like this. |
| Line 3: | Line 3: |
| == General == Poolmanager : poolman@pacs1 |
== Populate and access a local PacsPool == Script to populate a local PACS Pool starting from TM Files or Database |
| Line 6: | Line 6: |
| *[wiki:Self:PACS/Rsync Copying Pools with Rsync] *[wiki:Self:PACS/RsyncAdmin Data Pool Rsync Administration] |
{{{ from binstruct import * from spg import * from herschel.pacs.spg import ProcessObservationTask |
| Line 9: | Line 11: |
| == Available Pools == {{{ # ********** The official PACS Pool for Level 0 data of the ILT2 ************ # Status empty # Port 44530 hcss.dbpool.database = pacs_standard_products_fmilt@pacs1.mpe-garching.mpg.de |
# Populate Pacs Pool starting from file(s) f = "FILT_PhotFocus_filB_Aper1mm_20070616_12.tm" #c = populatePacsPool(filename=f, poolName ="Level0Test1", poolType="LocalStore", noObsid = True) #c = populatePacsPoolFromFiles(f, 111, "Level0Test1") |
| Line 16: | Line 16: |
| # ********** The official PACS calibration data Pool ****************** # Status : empty # Port : 44520 hcss.dbpool.cal = pacs_calibration_products@pacs1.mpe-garching.mpg.de |
# Populate Pacs Pool starting from Database #c = populatePacsPoolFromFiles(poolName = "Level0Test1", startDirectory="..") #c = populatePacsPoolFromDatabase(poolName = "Level0Test2", database="pacs_fm_ilt_3@pacs5") c = populatePacsPoolFromDatabase(536874110l, "pacs_fm_ilt_3@pacs5", "Level0Test2") # Open a "Store" for a certain PacsPool (you may specify many Pools) lstore = LocalStoreFactory.getStore("Level0Test2") store = ProductStorage() store.register(lstore) # # OPen the GUI and "go shopping" # result = browseProduct(store) # My result of shopping (much less then going with Tina) print result.size() # Get Observation Context # obsCont = result[0].product # Extract level 0 data level0 = obsCont.level0 print level0 # Extract Herschel Pacs Photometer AVEraged Blue data (frames) frames = level0.refs["HPPAVGB"].product.refs[0].product print frames # Extract Herschel Pacs OGSE HouseKeeping data hk = level0.refs["HPOGSEHK"].product.refs[0].product["HPOGSEHKS"] # What does it contain ? print hk # Extract a certain value print hk["BB1_Heat_Power"] # Export to Fits (for Helmut, but may find it also in Pool) fa=FitsArchive() p=Product() p["ogse"] = hk fa.save("ogse.fits",p) p2 = fa.load("ogse.fits") print p2["ogse"]["BB1_Heat_Power"] # # Access data by command line # query = MetaQuery(ObservationContext, 'h', 'h.meta["obsid"].value == 536874110') results = store.select(query) print results.size() # obsCont = results[0].product level0 = obsCont.level0 print level0 frames = level0.refs["HPPAVGB"].product.refs[0].product print frames |
| Line 23: | Line 78: |
| == Populating the pacs_standard_products_fmilt Pool == Login : oper@pacs5 Set the Pool directors in .pcss/props : {{{ hcss.ia.pal.pool.simple.dir = /pacs/dbBck/PacsProductPools }}} Currently oper@pacs5 is writing the pool to our server computer pacs1 using nfs mounted disks! Automatic processing after an Observation has been finished is not yet implemented ! The following script is used for Bulk Processing : {{{ import sys from herschel.store.api import * from herschel.ia.pal.pool.lstore import * from herschel.ia.pal.pool.db import DbPool from java.util import Date obsid = 536870913l # Database from which we fetch the TmSourcePackets database = "pacs_fm_ilt_3@pacs5" # Logfile to trace the bulk processing logfile = open("BulkProcessing.log",'w') # Logfile to trace problems problemfile = open("BulkProcessingProblems.log",'w') # The first 1000 Observations for i in range (1000) : # try : startMessage = "Processing started : "+ str(obsid) + " - " + Date().toString() + "\n" print startMessage logfile.write(startMessage) logfile.flush() # # Actual Processing # stat = populatePacsPool(obsid= obsid, poolName = poolName, poolType=pType, database=database , hklimit = 100000, sclimit = 1000) # endMessage = "Processing finished : "+ str(obsid) + " - " + Date().toString() + "\n" print endMessage logfile.write(endMessage) logfile.flush() # obsid+=1 # except : exceptMessage = "Processing Exception : "+ str(obsid) +" - " +Date().toString() + "\n" print exceptMessage obsid+=1 logfile.write(exceptMessage) problemfile.write(exceptMessage) e,inf,tb = sys.exc_info() print inf logfile.write(str(inf)) problemfile.write(str(inf)) logfile.flush() problemfile.flush() # pass pass logfile.close() problemfile.close() }}} == Make the Pool available for all == Login : poolman@pacs1 Then : {{{ java -Xmx800m -Dproperty.path=$HCSS_PROPS herschel.ia.pal.pool.serial.PoolDaemon 44529 pacs_standard_products_fmilt }}} == Accessing the Pacs Product Pools == The direct way is the following : {{{ pool = CachedPool(SerialClientPool("pacs1.mpe-garching.mpg.de", 44530, "pacs_standard_products_fmilt")) storage = ProductStorage() storage.register(pool) # Browse the Pool with GUI result = browseProduct(storage) }}} Using the Poolmanager : In the <PCSS_ROOT>/config/storage.props the properties pointing to files containing the Pool definitions are located. One among them is the <stdprod> Pool : {{{ cat pcss/config/pooldefs/stdprod.py pool = CachedPool(SerialClientPool("pacs1.mpe-garching.mpg.de", 44530, "pacs_standard_products_fmilt")) }}} In the jide session you do : {{{ pool = PoolManager.getPool("stdprod") storage = ProductStorage() storage.register(pool) # Browse the Pool with GUI result = browseProduct(storage) # After filling your shopping basket you may retrieve the products prod=result[0].getProduct() }}} You may also define your own pools. In your .pcss directory you define a Pool (in this case PoolExample.py : {{{ pool = SimplePool.getInstance("PoolExample") }}} Now you may access it in your jide session : {{{ pool = PoolManager.getPool("PoolExample") }}} |
== Copy a Pool == *[wiki:Self:PACS/Rsync Copying Pools with Rsync] *[wiki:Self:PACS/RsyncAdmin Data Pool Rsync Administration] |
PACS Data Pools
Populate and access a local PacsPool
Script to populate a local PACS Pool starting from TM Files or Database
from binstruct import *
from spg import *
from herschel.pacs.spg import ProcessObservationTask
# Populate Pacs Pool starting from file(s)
f = "FILT_PhotFocus_filB_Aper1mm_20070616_12.tm"
#c = populatePacsPool(filename=f, poolName ="Level0Test1", poolType="LocalStore", noObsid = True)
#c = populatePacsPoolFromFiles(f, 111, "Level0Test1")
# Populate Pacs Pool starting from Database
#c = populatePacsPoolFromFiles(poolName = "Level0Test1", startDirectory="..")
#c = populatePacsPoolFromDatabase(poolName = "Level0Test2", database="pacs_fm_ilt_3@pacs5")
c = populatePacsPoolFromDatabase(536874110l, "pacs_fm_ilt_3@pacs5", "Level0Test2")
# Open a "Store" for a certain PacsPool (you may specify many Pools)
lstore = LocalStoreFactory.getStore("Level0Test2")
store = ProductStorage()
store.register(lstore)
#
# OPen the GUI and "go shopping"
#
result = browseProduct(store)
# My result of shopping (much less then going with Tina)
print result.size()
# Get Observation Context
#
obsCont = result[0].product
# Extract level 0 data
level0 = obsCont.level0
print level0
# Extract Herschel Pacs Photometer AVEraged Blue data (frames)
frames = level0.refs["HPPAVGB"].product.refs[0].product
print frames
# Extract Herschel Pacs OGSE HouseKeeping data
hk = level0.refs["HPOGSEHK"].product.refs[0].product["HPOGSEHKS"]
# What does it contain ?
print hk
# Extract a certain value
print hk["BB1_Heat_Power"]
# Export to Fits (for Helmut, but may find it also in Pool)
fa=FitsArchive()
p=Product()
p["ogse"] = hk
fa.save("ogse.fits",p)
p2 = fa.load("ogse.fits")
print p2["ogse"]["BB1_Heat_Power"]
#
# Access data by command line
#
query = MetaQuery(ObservationContext, 'h', 'h.meta["obsid"].value == 536874110')
results = store.select(query)
print results.size()
#
obsCont = results[0].product
level0 = obsCont.level0
print level0
frames = level0.refs["HPPAVGB"].product.refs[0].product
print frames
Copy a Pool
[wiki:PACS/Rsync Copying Pools with Rsync]
[wiki:PACS/RsyncAdmin Data Pool Rsync Administration]