001 package org.cumulus4j.store.datastoreversion.command;
002
003 import java.util.ArrayList;
004 import java.util.Comparator;
005 import java.util.HashMap;
006 import java.util.List;
007 import java.util.Map;
008 import java.util.Properties;
009 import java.util.Set;
010
011 import javax.jdo.PersistenceManager;
012 import javax.jdo.Query;
013
014 import org.cumulus4j.store.Cumulus4jPersistenceHandler;
015 import org.cumulus4j.store.Cumulus4jStoreManager;
016 import org.cumulus4j.store.EncryptionHandler;
017 import org.cumulus4j.store.IndexEntryAction;
018 import org.cumulus4j.store.ProgressInfo;
019 import org.cumulus4j.store.WorkInProgressException;
020 import org.cumulus4j.store.crypto.CryptoContext;
021 import org.cumulus4j.store.datastoreversion.AbstractDatastoreVersionCommand;
022 import org.cumulus4j.store.datastoreversion.CommandApplyParam;
023 import org.cumulus4j.store.model.ClassMeta;
024 import org.cumulus4j.store.model.DataEntry;
025 import org.cumulus4j.store.model.FieldMeta;
026 import org.cumulus4j.store.model.IndexEntry;
027 import org.cumulus4j.store.model.ObjectContainer;
028 import org.datanucleus.ClassLoaderResolver;
029 import org.datanucleus.metadata.AbstractMemberMetaData;
030 import org.datanucleus.store.ExecutionContext;
031 import org.slf4j.Logger;
032 import org.slf4j.LoggerFactory;
033
034 /**
035 * Delete all {@link IndexEntry}s from the datastore and then iterate all
036 * {@link DataEntry}s and re-index them.
037 * <p>
038 * TODO This class currently does not yet ensure that different keys are used. Thus,
039 * very likely the entire index uses only one single key. This should be improved
040 * in a future version.
041 * @author Marco หงุ่ยตระกูล-Schulze - marco at nightlabs dot de
042 */
043 public class RecreateIndex extends AbstractDatastoreVersionCommand
044 {
045 private static final Logger logger = LoggerFactory.getLogger(RecreateIndex.class);
046
047 @Override
048 public int getCommandVersion() {
049 return 1;
050 }
051
052 @Override
053 public boolean isFinal() {
054 return false;
055 }
056
057 @Override
058 public boolean isKeyStoreDependent() {
059 return true;
060 }
061
062 private CommandApplyParam commandApplyParam;
063 private Properties workInProgressStateProperties;
064 private CryptoContext cryptoContext;
065 private PersistenceManager pmIndex;
066 private PersistenceManager pmData;
067 private long keyStoreRefID;
068
069 private Set<Class<? extends IndexEntry>> indexEntryClasses;
070
071 @Override
072 public void apply(CommandApplyParam commandApplyParam) {
073 // The index only exists in the index-datastore (not in the index-datastore), hence we return immediately, if the
074 // current datastore is not the index-datastore.
075 this.commandApplyParam = commandApplyParam;
076 PersistenceManager pm = commandApplyParam.getPersistenceManager();
077 cryptoContext = commandApplyParam.getCryptoContext();
078 if (pm != cryptoContext.getPersistenceManagerForIndex())
079 return;
080
081 keyStoreRefID = cryptoContext.getKeyStoreRefID();
082 pmIndex = commandApplyParam.getCryptoContext().getPersistenceManagerForIndex();
083 pmData = commandApplyParam.getCryptoContext().getPersistenceManagerForData();
084 workInProgressStateProperties = commandApplyParam.getWorkInProgressStateProperties();
085
086 deleteIndex();
087 createIndex();
088 }
089
090 protected static final String PROPERTY_DELETE_COMPLETE = "delete.complete";
091 protected static final String PROPERTY_DELETE_FROM_INDEX_ENTRY_ID = "delete.fromIndexEntryID";
092 protected static final String PROPERTY_CREATE_FROM_DATA_ENTRY_ID = "create.fromDataEntryID";
093
094 protected void deleteIndex() {
095 logger.debug("deleteIndex: Entered.");
096 if (Boolean.parseBoolean(workInProgressStateProperties.getProperty(PROPERTY_DELETE_COMPLETE))) {
097 logger.debug("deleteIndex: PROPERTY_DELETE_COMPLETE == true => quit.");
098 return;
099 }
100
101 final long indexEntryBlockSize = 100;
102 Long maxIndexEntryIDObj = getMaxIndexEntryID();
103 if (maxIndexEntryIDObj == null) {
104 logger.debug("deleteIndex: There are no IndexEntry instances in the database => quit.");
105 }
106 else {
107 final long maxIndexEntryID = maxIndexEntryIDObj;
108 String fromIndexEntryStr = workInProgressStateProperties.getProperty(PROPERTY_DELETE_FROM_INDEX_ENTRY_ID);
109 long fromIndexEntryID;
110 if (fromIndexEntryStr != null) {
111 logger.info("deleteIndex: previous incomplete run found: fromIndexEntryStr={}", fromIndexEntryStr);
112 fromIndexEntryID = Long.parseLong(fromIndexEntryStr);
113 }
114 else {
115 final long minIndexEntryID = getMinIndexEntryID();
116 logger.info("deleteIndex: first run: minIndexEntryID={} maxIndexEntryID={}", minIndexEntryID, maxIndexEntryID);
117 fromIndexEntryID = minIndexEntryID;
118 }
119 while (fromIndexEntryID <= maxIndexEntryID - indexEntryBlockSize) {
120 long toIndexEntryIDExcl = fromIndexEntryID + indexEntryBlockSize;
121 deleteIndexForRange(fromIndexEntryID, toIndexEntryIDExcl);
122 fromIndexEntryID = toIndexEntryIDExcl;
123 if (commandApplyParam.isDatastoreVersionCommandApplyWorkInProgressTimeoutExceeded()) {
124 workInProgressStateProperties.setProperty(PROPERTY_DELETE_FROM_INDEX_ENTRY_ID, Long.toString(fromIndexEntryID));
125 throw new WorkInProgressException(new ProgressInfo());
126 }
127 }
128 deleteIndexForRange(fromIndexEntryID, null);
129 }
130
131 workInProgressStateProperties.setProperty(PROPERTY_DELETE_COMPLETE, Boolean.TRUE.toString());
132 logger.debug("deleteIndex: Leaving.");
133 }
134
135 protected void deleteIndexForRange(long fromIndexEntryIDIncl, Long toIndexEntryIDExcl) {
136 logger.info("deleteIndexForRange: Entered. fromIndexEntryIDIncl={} toIndexEntryIDExcl={}", fromIndexEntryIDIncl, toIndexEntryIDExcl);
137 List<IndexEntry> indexEntries = getIndexEntries(fromIndexEntryIDIncl, toIndexEntryIDExcl);
138 pmIndex.deletePersistentAll(indexEntries);
139 logger.info("deleteIndexForRange: Leaving. fromIndexEntryIDIncl={} toIndexEntryIDExcl={}", fromIndexEntryIDIncl, toIndexEntryIDExcl);
140 }
141
142 protected List<IndexEntry> getIndexEntries(long fromIndexEntryIDIncl, Long toIndexEntryIDExcl) {
143 List<IndexEntry> result = new ArrayList<IndexEntry>();
144 for (Class<? extends IndexEntry> indexEntryClass : getIndexEntryClasses()) {
145 result.addAll(getIndexEntries(indexEntryClass, fromIndexEntryIDIncl, toIndexEntryIDExcl));
146 }
147 return result;
148 }
149
150 protected List<IndexEntry> getIndexEntries(Class<? extends IndexEntry> indexEntryClass, long fromIndexEntryIDIncl, Long toIndexEntryIDExcl) {
151 Query q = pmIndex.newQuery(indexEntryClass);
152 StringBuilder filter = new StringBuilder();
153 Map<String, Object> params = new HashMap<String, Object>(2);
154
155 filter.append("this.keyStoreRefID == :keyStoreRefID");
156 params.put("keyStoreRefID", keyStoreRefID);
157
158 if (fromIndexEntryIDIncl > 0) { // required for GAE, because it throws an exception when querying for ID >= 0, saying that ID == 0 is illegal.
159 filter.append(" && this.indexEntryID >= :fromIndexEntryIDIncl");
160 params.put("fromIndexEntryIDIncl", fromIndexEntryIDIncl);
161 }
162
163 if (toIndexEntryIDExcl != null) {
164 filter.append(" && this.indexEntryID < :toIndexEntryIDExcl");
165 params.put("toIndexEntryIDExcl", toIndexEntryIDExcl);
166 }
167 q.setFilter(filter.toString());
168 q.setOrdering("this.indexEntryID ASC");
169
170 @SuppressWarnings("unchecked")
171 List<IndexEntry> result = new ArrayList<IndexEntry>((List<IndexEntry>) q.executeWithMap(params));
172 q.closeAll();
173 return result;
174 }
175
176 protected void createIndex()
177 {
178 final long dataEntryBlockSize = 100;
179 long fromDataEntryID;
180 String fromDataEntryIDStr = workInProgressStateProperties.getProperty(PROPERTY_CREATE_FROM_DATA_ENTRY_ID);
181 final long maxDataEntryID = getMaxDataEntryID();
182 if (fromDataEntryIDStr != null) {
183 fromDataEntryID = Long.parseLong(fromDataEntryIDStr);
184 }
185 else {
186 final long minDataEntryID = getMinDataEntryID();
187 fromDataEntryID = minDataEntryID;
188 }
189 while (fromDataEntryID <= maxDataEntryID - dataEntryBlockSize) {
190 long toDataEntryIDExcl = fromDataEntryID + dataEntryBlockSize;
191 createIndexForRange(fromDataEntryID, toDataEntryIDExcl);
192 fromDataEntryID = toDataEntryIDExcl;
193
194 if (commandApplyParam.isDatastoreVersionCommandApplyWorkInProgressTimeoutExceeded()) {
195 workInProgressStateProperties.setProperty(PROPERTY_CREATE_FROM_DATA_ENTRY_ID, Long.toString(fromDataEntryID));
196 throw new WorkInProgressException(new ProgressInfo());
197 }
198 }
199 createIndexForRange(fromDataEntryID, null);
200 }
201
202 protected void createIndexForRange(long fromDataEntryIDIncl, Long toDataEntryIDExcl) {
203 ExecutionContext ec = cryptoContext.getExecutionContext();
204 ClassLoaderResolver clr = ec.getClassLoaderResolver();
205 Cumulus4jStoreManager storeManager = commandApplyParam.getStoreManager();
206 EncryptionHandler encryptionHandler = storeManager.getEncryptionHandler();
207 Cumulus4jPersistenceHandler persistenceHandler = storeManager.getPersistenceHandler();
208 IndexEntryAction addIndexEntryAction = persistenceHandler.getAddIndexEntryAction();
209 List<DataEntryWithClassName> l = getDataEntries(fromDataEntryIDIncl, toDataEntryIDExcl);
210 for (DataEntryWithClassName dataEntryWithClassName : l) {
211 long dataEntryID = dataEntryWithClassName.getDataEntry().getDataEntryID();
212 Class<?> clazz = clr.classForName(dataEntryWithClassName.getClassName());
213 ClassMeta classMeta = storeManager.getClassMeta(ec, clazz);
214 ObjectContainer objectContainer = encryptionHandler.decryptDataEntry(cryptoContext, dataEntryWithClassName.getDataEntry());
215 for (Map.Entry<Long, Object> me : objectContainer.getFieldID2value().entrySet()) {
216 long fieldID = me.getKey();
217 Object fieldValue = me.getValue();
218 FieldMeta fieldMeta = classMeta.getFieldMeta(fieldID);
219 AbstractMemberMetaData dnMemberMetaData = fieldMeta.getDataNucleusMemberMetaData(ec);
220 addIndexEntryAction.perform(cryptoContext, dataEntryID, fieldMeta, dnMemberMetaData, classMeta, fieldValue);
221 }
222 }
223 }
224
225 protected static final class DataEntryWithClassName {
226 private DataEntry dataEntry;
227 private String className;
228
229 public DataEntryWithClassName(DataEntry dataEntry, String className) {
230 if (dataEntry == null)
231 throw new IllegalArgumentException("dataEntry == null");
232 if (className == null)
233 throw new IllegalArgumentException("className == null");
234 this.dataEntry = dataEntry;
235 this.className = className;
236 }
237 /**
238 * Get the {@link DataEntry}.
239 * @return the {@link DataEntry}. Never <code>null</code>.
240 */
241 public DataEntry getDataEntry() {
242 return dataEntry;
243 }
244 /**
245 * Get the fully qualified class name of the persistence-capable object represented by
246 * {@link #dataEntry}.
247 * @return the fully qualified class name. Never <code>null</code>.
248 */
249 public String getClassName() {
250 return className;
251 }
252 }
253
254 protected List<DataEntryWithClassName> getDataEntries(long fromDataEntryIDIncl, Long toDataEntryIDExcl) {
255 Query q = pmData.newQuery(DataEntry.class);
256 q.declareVariables(ClassMeta.class.getName() + " classMeta");
257 q.setResult("this, classMeta.packageName, classMeta.simpleClassName");
258 StringBuilder filter = new StringBuilder();
259 Map<String, Object> params = new HashMap<String, Object>(2);
260
261 filter.append("this.classMeta_classID == classMeta.classID");
262 filter.append(" && this.keyStoreRefID == :keyStoreRefID");
263 params.put("keyStoreRefID", keyStoreRefID);
264
265 if (fromDataEntryIDIncl > 0) { // required for GAE, because it throws an exception when querying for ID >= 0, saying that ID == 0 is illegal.
266 filter.append(" && this.dataEntryID >= :fromDataEntryIDIncl");
267 params.put("fromDataEntryIDIncl", fromDataEntryIDIncl);
268 }
269
270 if (toDataEntryIDExcl != null) {
271 filter.append(" && this.dataEntryID < :toDataEntryIDExcl");
272 params.put("toDataEntryIDExcl", toDataEntryIDExcl);
273 }
274 q.setFilter(filter.toString());
275 q.setOrdering("this.dataEntryID ASC");
276
277 @SuppressWarnings("unchecked")
278 List<Object[]> l = (List<Object[]>) q.executeWithMap(params);
279 List<DataEntryWithClassName> result = new ArrayList<DataEntryWithClassName>(l.size());
280 for (Object[] row : l) {
281 if (row.length != 3)
282 throw new IllegalStateException(String.format("row.length == %s != 3", row.length));
283
284 result.add(new DataEntryWithClassName(
285 (DataEntry)row[0],
286 ClassMeta.getClassName((String)row[1], (String)row[2])
287 ));
288 }
289 q.closeAll();
290 return result;
291 }
292
293 protected long getMinDataEntryID() {
294 Long result = getMinMaxDataEntryID("min");
295 if (result == null)
296 return 0;
297 return result;
298 }
299
300 protected long getMaxDataEntryID() {
301 Long result = getMinMaxDataEntryID("max");
302 if (result == null)
303 return 0;
304 return result;
305 }
306
307 protected Long getMinMaxDataEntryID(String minMax) {
308 Query q = pmData.newQuery(DataEntry.class);
309 q.setResult(minMax + "(this.dataEntryID)");
310 Long result = (Long) q.execute();
311 return result;
312 }
313
314 protected Long getMinIndexEntryID() {
315 return getMinMaxIndexEntryID("min", new Comparator<Long>() {
316 @Override
317 public int compare(Long o1, Long o2) {
318 return o2.compareTo(o1);
319 }
320 });
321 }
322
323 protected Long getMaxIndexEntryID() {
324 return getMinMaxIndexEntryID("max", new Comparator<Long>() {
325 @Override
326 public int compare(Long o1, Long o2) {
327 return o1.compareTo(o2);
328 }
329 });
330 }
331
332 protected Long getMinMaxIndexEntryID(String minMax, Comparator<Long> comparator) {
333 Long result = null;
334 for (Class<? extends IndexEntry> indexEntryClass : getIndexEntryClasses()) {
335 Long minMaxIndexEntryID = getMinMaxIndexEntryID(indexEntryClass, minMax);
336 if (minMaxIndexEntryID != null) {
337 if (result == null || comparator.compare(result, minMaxIndexEntryID) < 0)
338 result = minMaxIndexEntryID;
339 }
340 }
341 return result;
342 }
343
344 protected Long getMinMaxIndexEntryID(Class<? extends IndexEntry> indexEntryClass, String minMax) {
345 Query q = pmIndex.newQuery(indexEntryClass);
346 q.setResult(minMax + "(this.indexEntryID)");
347 Long result = (Long) q.execute();
348 return result;
349 }
350
351 protected Set<Class<? extends IndexEntry>> getIndexEntryClasses() {
352 if (indexEntryClasses == null)
353 indexEntryClasses = ((Cumulus4jStoreManager)cryptoContext.getExecutionContext().getStoreManager()).getIndexFactoryRegistry().getIndexEntryClasses();
354
355 return indexEntryClasses;
356 }
357 }