001 /*
002 * Cumulus4j - Securing your data in the cloud - http://cumulus4j.org
003 * Copyright (C) 2011 NightLabs Consulting GmbH
004 *
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU Affero General Public License as
007 * published by the Free Software Foundation, either version 3 of the
008 * License, or (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
013 * GNU Affero General Public License for more details.
014 *
015 * You should have received a copy of the GNU Affero General Public License
016 * along with this program. If not, see <http://www.gnu.org/licenses/>.
017 */
018 package org.cumulus4j.store.query;
019
020 import java.util.Collection;
021 import java.util.Deque;
022 import java.util.HashMap;
023 import java.util.HashSet;
024 import java.util.Iterator;
025 import java.util.LinkedList;
026 import java.util.List;
027 import java.util.Map;
028 import java.util.Set;
029
030 import javax.jdo.PersistenceManager;
031
032 import org.cumulus4j.store.Cumulus4jStoreManager;
033 import org.cumulus4j.store.EncryptionHandler;
034 import org.cumulus4j.store.PersistenceManagerConnection;
035 import org.cumulus4j.store.crypto.CryptoContext;
036 import org.cumulus4j.store.model.ClassMeta;
037 import org.cumulus4j.store.model.ClassMetaDAO;
038 import org.cumulus4j.store.model.DataEntry;
039 import org.cumulus4j.store.model.EmbeddedClassMeta;
040 import org.cumulus4j.store.query.eval.AbstractExpressionEvaluator;
041 import org.cumulus4j.store.query.eval.AndExpressionEvaluator;
042 import org.cumulus4j.store.query.eval.ComparisonExpressionEvaluator;
043 import org.cumulus4j.store.query.eval.InvokeExpressionEvaluator;
044 import org.cumulus4j.store.query.eval.LiteralEvaluator;
045 import org.cumulus4j.store.query.eval.NotExpressionEvaluator;
046 import org.cumulus4j.store.query.eval.OrExpressionEvaluator;
047 import org.cumulus4j.store.query.eval.ParameterExpressionEvaluator;
048 import org.cumulus4j.store.query.eval.PrimaryExpressionEvaluator;
049 import org.cumulus4j.store.query.eval.ResultDescriptor;
050 import org.cumulus4j.store.query.eval.SubqueryExpressionEvaluator;
051 import org.cumulus4j.store.query.eval.VariableExpressionEvaluator;
052 import org.datanucleus.ClassLoaderResolver;
053 import org.datanucleus.identity.IdentityUtils;
054 import org.datanucleus.metadata.AbstractClassMetaData;
055 import org.datanucleus.query.QueryUtils;
056 import org.datanucleus.query.compiler.QueryCompilation;
057 import org.datanucleus.query.expression.DyadicExpression;
058 import org.datanucleus.query.expression.Expression;
059 import org.datanucleus.query.expression.InvokeExpression;
060 import org.datanucleus.query.expression.Literal;
061 import org.datanucleus.query.expression.ParameterExpression;
062 import org.datanucleus.query.expression.PrimaryExpression;
063 import org.datanucleus.query.expression.SubqueryExpression;
064 import org.datanucleus.query.expression.VariableExpression;
065 import org.datanucleus.query.symbol.Symbol;
066 import org.datanucleus.store.ExecutionContext;
067 import org.datanucleus.store.query.Query;
068
069 /**
070 * API-agnostic query implementation. An instance of this class performs the actual query.
071 * It is used by both APIs, JDO and JPA.
072 *
073 * @author Marco หงุ่ยตระกูล-Schulze - marco at nightlabs dot de
074 */
075 public abstract class QueryEvaluator
076 {
077 /** Name under which any set of results are stored in the state map. Used for aggregation. */
078 public static final String RESULTS_SET = "DATANUCLEUS_RESULTS_SET";
079
080 private final String language;
081
082 private String candidateAlias = "this";
083
084 /** Underlying "string-based" query. */
085 private Query query;
086
087 /** Compilation of the underlying query, that we are evaluating. */
088 private QueryCompilation compilation;
089
090 /** Map of input parameter values, keyed by the parameter name. */
091 private Map<String, Object> parameterValues;
092
093 /** Positional parameter that we are up to (-1 implies not being used). */
094 private Map<Integer, Symbol> paramSymbolByPosition = null;
095
096 /** Map of state symbols for the query evaluation. */
097 private Map<String, Object> state;
098
099 private ClassLoaderResolver clr;
100
101 private ExecutionContext ec;
102
103 private Cumulus4jStoreManager storeManager;
104
105 private CryptoContext cryptoContext;
106
107 private PersistenceManagerConnection pmConn;
108
109 private EncryptionHandler encryptionHandler;
110
111 private boolean complete = true;
112
113 private Map<Symbol, EmbeddedClassMeta> symbol2ValueTypeEmbeddedClassMeta = null;
114
115 /**
116 * @param language Query language (JDOQL, JPQL, etc)
117 * @param compilation generic compilation
118 * @param parameterValues Input values for the params
119 * @param clr ClassLoader resolver
120 * @param pmConn our <b>backend</b>-<code>PersistenceManager</code> connection(s).
121 * @param cryptoContext TODO
122 */
123 public QueryEvaluator(
124 String language, Query query, QueryCompilation compilation, Map<String, Object> parameterValues,
125 ClassLoaderResolver clr, PersistenceManagerConnection pmConn, CryptoContext cryptoContext)
126 {
127 this.language = language;
128 this.query = query;
129 this.compilation = compilation;
130 this.parameterValues = parameterValues;
131 this.clr = clr;
132 this.ec = query.getExecutionContext();
133 this.storeManager = (Cumulus4jStoreManager) query.getStoreManager();
134 this.pmConn = pmConn;
135 this.cryptoContext = cryptoContext;
136 this.encryptionHandler = storeManager.getEncryptionHandler();
137
138 this.candidateAlias = (compilation.getCandidateAlias() != null ? compilation.getCandidateAlias() : this.candidateAlias);
139
140 state = new HashMap<String, Object>();
141 state.put(this.candidateAlias, query.getCandidateClass());
142
143 if (parameterValues != null && !parameterValues.isEmpty()) {
144 Object paramKey = parameterValues.keySet().iterator().next();
145 if (paramKey instanceof Integer) {
146 paramSymbolByPosition = new HashMap<Integer, Symbol>();
147 }
148 }
149 }
150
151 public boolean isComplete() {
152 return complete;
153 }
154
155 public void setIncomplete() {
156 this.complete = false;
157 }
158
159 public String getLanguage() {
160 return language;
161 }
162
163 public String getCandidateAlias() {
164 return candidateAlias;
165 }
166
167 public Query getQuery() {
168 return query;
169 }
170
171 public QueryCompilation getCompilation() {
172 return compilation;
173 }
174
175 public Map<String, Object> getParameterValues() {
176 return parameterValues;
177 }
178
179 public Map<String, Object> getState() {
180 return state;
181 }
182
183 public ClassLoaderResolver getClassLoaderResolver() {
184 return clr;
185 }
186
187 public ExecutionContext getExecutionContext() {
188 return ec;
189 }
190
191 public Cumulus4jStoreManager getStoreManager() {
192 return storeManager;
193 }
194
195 public PersistenceManagerConnection getPersistenceManagerConnection() {
196 return pmConn;
197 }
198
199 public PersistenceManager getPersistenceManagerForData() {
200 return pmConn.getDataPM();
201 }
202
203 public PersistenceManager getPersistenceManagerForIndex() {
204 return pmConn.getIndexPM();
205 }
206
207 public EncryptionHandler getEncryptionHandler() {
208 return encryptionHandler;
209 }
210
211 private Deque<ResultDescriptor> resultDescriptors = new LinkedList<ResultDescriptor>();
212
213 /**
214 * Push a {@link ResultDescriptor} onto the stack.
215 * @param resultDescriptor the descriptor to be pushed.
216 */
217 public void pushResultDescriptor(ResultDescriptor resultDescriptor)
218 {
219 resultDescriptors.push(resultDescriptor);
220 }
221
222 /**
223 * Pop a {@link ResultDescriptor} from the stack.
224 * @return the popped descriptor (which is the last one pushed).
225 */
226 public ResultDescriptor popResultDescriptor()
227 {
228 return resultDescriptors.pop();
229 }
230
231 public ClassMeta getValueTypeClassMeta(Symbol symbol, boolean throwExceptionIfNotFound) {
232 ClassMeta classMeta = getValueTypeEmbeddedClassMeta(symbol);
233 if (classMeta == null) {
234 Class<?> clazz = getValueType(symbol, throwExceptionIfNotFound);
235 classMeta = getStoreManager().getClassMeta(getExecutionContext(), clazz);
236 }
237 return classMeta;
238 }
239
240 public EmbeddedClassMeta getValueTypeEmbeddedClassMeta(Symbol symbol) {
241 if (symbol == null)
242 throw new IllegalArgumentException("symbol == null");
243
244 if (symbol2ValueTypeEmbeddedClassMeta == null)
245 return null;
246
247 return symbol2ValueTypeEmbeddedClassMeta.get(symbol);
248 }
249
250 public void registerValueTypeEmbeddedClassMeta(Symbol symbol, EmbeddedClassMeta embeddedClassMeta) {
251 if (symbol == null)
252 throw new IllegalArgumentException("symbol == null");
253
254 if (embeddedClassMeta == null)
255 return;
256
257 if (symbol2ValueTypeEmbeddedClassMeta == null)
258 symbol2ValueTypeEmbeddedClassMeta = new HashMap<Symbol, EmbeddedClassMeta>();
259
260 symbol2ValueTypeEmbeddedClassMeta.put(symbol, embeddedClassMeta);
261 }
262
263 /**
264 * Get a <code>Symbol</code>'s {@link Symbol#getValueType() valueType} by taking {@link ResultDescriptor}s into account.
265 * Delegates to {@link #getValueType(Symbol, boolean)} with <code>throwExceptionIfNotFound == true</code>.
266 *
267 * @param symbol the symbol whose {@link Symbol#getValueType() valueType} should be resolved.
268 * @return the type - never <code>null</code>.
269 * @see #getValueType(Symbol, boolean)
270 */
271 public Class<?> getValueType(Symbol symbol)
272 {
273 return getValueType(symbol, true);
274 }
275
276 /**
277 * <p>
278 * Get a <code>Symbol</code>'s {@link Symbol#getValueType() valueType} by taking {@link ResultDescriptor}s into account.
279 * </p>
280 * <p>
281 * This method (or alternatively {@link #getValueType(Symbol)}) should always be used instead of directly
282 * accessing {@link Symbol#getValueType()}!!! This allows for implicit variables (which are not declared).
283 * </p>
284 * <p>
285 * This method first checks, if {@link Symbol#getValueType()} returns a value and if so, returns it. Otherwise
286 * it searches the stack of {@link ResultDescriptor}s (maintained via {@link #pushResultDescriptor(ResultDescriptor)}
287 * and {@link #popResultDescriptor()}) and returns the first found {@link ResultDescriptor#getResultType()}.
288 * </p>
289 *
290 * @param symbol the symbol whose {@link Symbol#getValueType() valueType} should be resolved.
291 * @param throwExceptionIfNotFound whether to throw an {@link IllegalStateException} [exception type might be changed without notice!],
292 * if the type cannot be resolved. If <code>false</code> this method returns <code>null</code> instead.
293 * @return the type or <code>null</code>, if not resolvable and <code>throwExceptionIfNotFound == false</code>.
294 * @see #getValueType(Symbol)
295 * @see #pushResultDescriptor(ResultDescriptor)
296 * @see #popResultDescriptor()
297 */
298 public Class<?> getValueType(Symbol symbol, boolean throwExceptionIfNotFound)
299 {
300 if (symbol.getValueType() != null)
301 return symbol.getValueType();
302
303 for (ResultDescriptor resultDescriptor : resultDescriptors) {
304 if (symbol.equals(resultDescriptor.getSymbol()))
305 return resultDescriptor.getResultType();
306 }
307
308 if (symbol.getType() == Symbol.PARAMETER) {
309 // Cater for implicit parameters where the generic compilation doesn't have the type
310 if (paramSymbolByPosition != null) {
311 // Positional parameters
312 Iterator<Map.Entry<Integer, Symbol>> paramIter = paramSymbolByPosition.entrySet().iterator();
313 while (paramIter.hasNext()) {
314 Map.Entry<Integer, Symbol> entry = paramIter.next();
315 if (entry.getValue() == symbol) {
316 return parameterValues.get(entry.getKey()).getClass();
317 }
318 }
319
320 Integer nextPos = paramSymbolByPosition.size();
321 Object value = parameterValues.get(nextPos);
322 paramSymbolByPosition.put(nextPos, symbol);
323 return value.getClass();
324 }
325 else {
326 if (parameterValues.containsKey(symbol.getQualifiedName())) {
327 return parameterValues.get(symbol.getQualifiedName()).getClass();
328 }
329 }
330 }
331 if (throwExceptionIfNotFound)
332 throw new IllegalStateException("Could not determine the resultType of symbol \"" + symbol + "\"! If this is a variable, you might want to declare it.");
333
334 return null;
335 }
336
337 protected abstract Collection<Object> evaluateSubquery(
338 Query subquery, QueryCompilation compilation, Object outerCandidate
339 );
340
341 public List<Object> execute()
342 {
343 Class<?> candidateClass = query.getCandidateClass();
344 boolean withSubclasses = query.isSubclasses();
345 Set<ClassMeta> candidateClassMetas = QueryHelper.getCandidateClassMetas(storeManager, ec, candidateClass, withSubclasses);
346
347 // TODO I copied this from the JavaQueryEvaluator, but I'm not sure, whether we need this. Need to talk with Andy. Marco.
348 // ...or analyse it ourselves (step through)...
349 String[] subqueryAliases = compilation.getSubqueryAliases();
350 if (subqueryAliases != null) {
351 for (int i=0; i<subqueryAliases.length; ++i) {
352 // Evaluate subquery first
353 Query subquery = query.getSubqueryForVariable(subqueryAliases[i]).getQuery();
354 QueryCompilation subqueryCompilation = compilation.getCompilationForSubquery(subqueryAliases[i]);
355
356 Collection<Object> subqueryResult = evaluateSubquery(subquery, subqueryCompilation, null);
357
358 if (QueryUtils.queryReturnsSingleRow(subquery)) {
359 Iterator<Object> subqueryIterator = subqueryResult.iterator();
360 if (!subqueryIterator.hasNext()) // TODO simply use null in this case?!????
361 throw new IllegalStateException("Subquery is expected to return a single row, but it returned an empty collection!");
362
363 state.put(subqueryAliases[i], subqueryIterator.next());
364
365 if (subqueryIterator.hasNext())
366 throw new IllegalStateException("Subquery is expected to return only a single row, but it returned more than one!");
367 }
368 else
369 state.put(subqueryAliases[i], subqueryResult);
370 }
371 }
372
373 if (compilation.getExprFilter() == null) {
374 // No filter - we want all that match the candidate classes.
375 return QueryHelper.getAllPersistentObjectsForCandidateClasses(cryptoContext, getPersistenceManagerForData(), candidateClassMetas);
376 }
377 else {
378 expressionEvaluator = createExpressionEvaluatorTree(compilation.getExprFilter());
379 Symbol resultSymbol = getCompilation().getSymbolTable().getSymbol(getCandidateAlias());
380 if (resultSymbol == null)
381 throw new IllegalStateException("getCompilation().getSymbolTable().getSymbol(getCandidateAlias()) returned null! getCandidateAlias()==\"" + getCandidateAlias() + "\"");
382
383 return expressionEvaluator.queryResultObjects(new ResultDescriptor(resultSymbol, null));
384 }
385 }
386
387 private AbstractExpressionEvaluator<?> expressionEvaluator;
388
389 public AbstractExpressionEvaluator<?> getExpressionEvaluator() {
390 return expressionEvaluator;
391 }
392
393 private AbstractExpressionEvaluator<?> createExpressionEvaluatorTree(Expression expression)
394 {
395 return createExpressionEvaluatorTreeRecursive(null, expression);
396 }
397
398 private AbstractExpressionEvaluator<?> createExpressionEvaluatorTreeRecursive(AbstractExpressionEvaluator<?> parent, Expression expression)
399 {
400 AbstractExpressionEvaluator<?> eval = createExpressionEvaluator(parent, expression);
401
402 if (expression.getLeft() != null) {
403 AbstractExpressionEvaluator<?> childEval = createExpressionEvaluatorTreeRecursive(eval, expression.getLeft());
404 eval.setLeft(childEval);
405 }
406
407 if (expression.getRight() != null) {
408 AbstractExpressionEvaluator<?> childEval = createExpressionEvaluatorTreeRecursive(eval, expression.getRight());
409 eval.setRight(childEval);
410 }
411
412 return eval;
413 }
414
415 private AbstractExpressionEvaluator<?> createExpressionEvaluator(
416 AbstractExpressionEvaluator<?> parent,
417 Expression expr
418 )
419 {
420 if (expr instanceof DyadicExpression) {
421 DyadicExpression expression = (DyadicExpression) expr;
422 if (
423 Expression.OP_EQ == expression.getOperator() ||
424 Expression.OP_NOTEQ == expression.getOperator() ||
425 Expression.OP_LT == expression.getOperator() ||
426 Expression.OP_LTEQ == expression.getOperator() ||
427 Expression.OP_GT == expression.getOperator() ||
428 Expression.OP_GTEQ == expression.getOperator()
429 )
430 return new ComparisonExpressionEvaluator(this, parent, expression);
431 else if (Expression.OP_AND == expression.getOperator())
432 return new AndExpressionEvaluator(this, parent, expression);
433 else if (Expression.OP_OR == expression.getOperator())
434 return new OrExpressionEvaluator(this, parent, expression);
435 else if (Expression.OP_NOT == expression.getOperator())
436 return new NotExpressionEvaluator(this, parent, expression);
437 else
438 throw new UnsupportedOperationException("Unsupported operator for DyadicExpression: " + expr);
439 }
440
441 if (expr instanceof PrimaryExpression)
442 return new PrimaryExpressionEvaluator(this, parent, (PrimaryExpression) expr);
443
444 if (expr instanceof ParameterExpression)
445 return new ParameterExpressionEvaluator(this, parent, (ParameterExpression) expr);
446
447 if (expr instanceof Literal)
448 return new LiteralEvaluator(this, parent, (Literal) expr);
449
450 if (expr instanceof InvokeExpression)
451 return new InvokeExpressionEvaluator(this, parent, (InvokeExpression) expr);
452
453 if (expr instanceof VariableExpression)
454 return new VariableExpressionEvaluator(this, parent, (VariableExpression) expr);
455
456 if (expr instanceof SubqueryExpression)
457 return new SubqueryExpressionEvaluator(this, parent, (SubqueryExpression) expr);
458
459 throw new UnsupportedOperationException("Don't know what to do with this expression: " + expr);
460 }
461
462 public Object getObjectForDataEntry(DataEntry dataEntry)
463 {
464 return getObjectForClassMetaAndObjectIDString(dataEntry.getClassMeta(), dataEntry.getObjectID());
465 }
466
467 public Object getObjectForClassMetaAndObjectIDString(ClassMeta classMeta, String objectIDString)
468 {
469 AbstractClassMetaData cmd = classMeta.getDataNucleusClassMetaData(ec);
470 return IdentityUtils.getObjectFromIdString(objectIDString, cmd, ec, true);
471 }
472
473 public Set<Long> getAllDataEntryIDsForCandidateClasses(Set<ClassMeta> candidateClassMetas)
474 {
475 javax.jdo.Query q = getPersistenceManagerForData().newQuery(DataEntry.class);
476 q.setResult("this.dataEntryID");
477
478
479 Map<String, Object> queryParams = new HashMap<String, Object>();
480 StringBuilder filter = new StringBuilder();
481
482 filter.append("this.keyStoreRefID == :keyStoreRefID && ");
483 queryParams.put("keyStoreRefID", cryptoContext.getKeyStoreRefID());
484
485 filter.append(ClassMetaDAO.getMultiClassMetaOrFilterPart(queryParams, candidateClassMetas));
486
487 q.setFilter(filter.toString());
488
489 @SuppressWarnings("unchecked")
490 Collection<Long> allDataEntryIDs = (Collection<Long>) q.executeWithMap(queryParams);
491 Set<Long> result = new HashSet<Long>(allDataEntryIDs);
492 q.closeAll();
493 return result;
494 }
495
496 public CryptoContext getCryptoContext() {
497 return cryptoContext;
498 }
499 }