001/*
002 * Copyright 2013-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2013-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.OutputStream;
026import java.util.Collections;
027import java.util.LinkedHashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.TreeMap;
031import java.util.concurrent.atomic.AtomicLong;
032
033import com.unboundid.asn1.ASN1OctetString;
034import com.unboundid.ldap.sdk.Attribute;
035import com.unboundid.ldap.sdk.DN;
036import com.unboundid.ldap.sdk.Filter;
037import com.unboundid.ldap.sdk.LDAPConnection;
038import com.unboundid.ldap.sdk.LDAPException;
039import com.unboundid.ldap.sdk.LDAPSearchException;
040import com.unboundid.ldap.sdk.ResultCode;
041import com.unboundid.ldap.sdk.SearchRequest;
042import com.unboundid.ldap.sdk.SearchResult;
043import com.unboundid.ldap.sdk.SearchResultEntry;
044import com.unboundid.ldap.sdk.SearchResultReference;
045import com.unboundid.ldap.sdk.SearchResultListener;
046import com.unboundid.ldap.sdk.SearchScope;
047import com.unboundid.ldap.sdk.Version;
048import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
049import com.unboundid.util.Debug;
050import com.unboundid.util.LDAPCommandLineTool;
051import com.unboundid.util.StaticUtils;
052import com.unboundid.util.ThreadSafety;
053import com.unboundid.util.ThreadSafetyLevel;
054import com.unboundid.util.args.ArgumentException;
055import com.unboundid.util.args.ArgumentParser;
056import com.unboundid.util.args.DNArgument;
057import com.unboundid.util.args.IntegerArgument;
058import com.unboundid.util.args.StringArgument;
059
060
061
062/**
063 * This class provides a tool that may be used to identify references to entries
064 * that do not exist.  This tool can be useful for verifying existing data in
065 * directory servers that provide support for referential integrity.
066 * <BR><BR>
067 * All of the necessary information is provided using command line arguments.
068 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
069 * class, as well as the following additional arguments:
070 * <UL>
071 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
072 *       for the searches.  At least one base DN must be provided.</LI>
073 *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
074 *       that is expected to contain references to other entries.  This
075 *       attribute should be indexed for equality searches, and its values
076 *       should be DNs.  At least one attribute must be provided.</LI>
077 *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
078 *       to find entries with references to other entries should use the simple
079 *       paged results control to iterate across entries in fixed-size pages
080 *       rather than trying to use a single search to identify all entries that
081 *       reference other entries.</LI>
082 * </UL>
083 */
084@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
085public final class IdentifyReferencesToMissingEntries
086       extends LDAPCommandLineTool
087       implements SearchResultListener
088{
089  /**
090   * The serial version UID for this serializable class.
091   */
092  private static final long serialVersionUID = 1981894839719501258L;
093
094
095
096  // The number of entries examined so far.
097  private final AtomicLong entriesExamined;
098
099  // The argument used to specify the base DNs to use for searches.
100  private DNArgument baseDNArgument;
101
102  // The argument used to specify the search page size.
103  private IntegerArgument pageSizeArgument;
104
105  // The connection to use for retrieving referenced entries.
106  private LDAPConnection getReferencedEntriesConnection;
107
108  // A map with counts of missing references by attribute type.
109  private final Map<String,AtomicLong> missingReferenceCounts;
110
111  // The names of the attributes for which to find missing references.
112  private String[] attributes;
113
114  // The argument used to specify the attributes for which to find missing
115  // references.
116  private StringArgument attributeArgument;
117
118
119
120  /**
121   * Parse the provided command line arguments and perform the appropriate
122   * processing.
123   *
124   * @param  args  The command line arguments provided to this program.
125   */
126  public static void main(final String... args)
127  {
128    final ResultCode resultCode = main(args, System.out, System.err);
129    if (resultCode != ResultCode.SUCCESS)
130    {
131      System.exit(resultCode.intValue());
132    }
133  }
134
135
136
137  /**
138   * Parse the provided command line arguments and perform the appropriate
139   * processing.
140   *
141   * @param  args       The command line arguments provided to this program.
142   * @param  outStream  The output stream to which standard out should be
143   *                    written.  It may be {@code null} if output should be
144   *                    suppressed.
145   * @param  errStream  The output stream to which standard error should be
146   *                    written.  It may be {@code null} if error messages
147   *                    should be suppressed.
148   *
149   * @return A result code indicating whether the processing was successful.
150   */
151  public static ResultCode main(final String[] args,
152                                final OutputStream outStream,
153                                final OutputStream errStream)
154  {
155    final IdentifyReferencesToMissingEntries tool =
156         new IdentifyReferencesToMissingEntries(outStream, errStream);
157    return tool.runTool(args);
158  }
159
160
161
162  /**
163   * Creates a new instance of this tool.
164   *
165   * @param  outStream  The output stream to which standard out should be
166   *                    written.  It may be {@code null} if output should be
167   *                    suppressed.
168   * @param  errStream  The output stream to which standard error should be
169   *                    written.  It may be {@code null} if error messages
170   *                    should be suppressed.
171   */
172  public IdentifyReferencesToMissingEntries(final OutputStream outStream,
173                                            final OutputStream errStream)
174  {
175    super(outStream, errStream);
176
177    baseDNArgument = null;
178    pageSizeArgument = null;
179    attributeArgument = null;
180    getReferencedEntriesConnection = null;
181
182    entriesExamined = new AtomicLong(0L);
183    missingReferenceCounts = new TreeMap<String, AtomicLong>();
184  }
185
186
187
188  /**
189   * Retrieves the name of this tool.  It should be the name of the command used
190   * to invoke this tool.
191   *
192   * @return  The name for this tool.
193   */
194  @Override()
195  public String getToolName()
196  {
197    return "identify-references-to-missing-entries";
198  }
199
200
201
202  /**
203   * Retrieves a human-readable description for this tool.
204   *
205   * @return  A human-readable description for this tool.
206   */
207  @Override()
208  public String getToolDescription()
209  {
210    return "This tool may be used to identify entries containing one or more " +
211         "attributes which reference entries that do not exist.  This may " +
212         "require the ability to perform unindexed searches and/or the " +
213         "ability to use the simple paged results control.";
214  }
215
216
217
218  /**
219   * Retrieves a version string for this tool, if available.
220   *
221   * @return  A version string for this tool, or {@code null} if none is
222   *          available.
223   */
224  @Override()
225  public String getToolVersion()
226  {
227    return Version.NUMERIC_VERSION_STRING;
228  }
229
230
231
232  /**
233   * Indicates whether this tool should provide support for an interactive mode,
234   * in which the tool offers a mode in which the arguments can be provided in
235   * a text-driven menu rather than requiring them to be given on the command
236   * line.  If interactive mode is supported, it may be invoked using the
237   * "--interactive" argument.  Alternately, if interactive mode is supported
238   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
239   * interactive mode may be invoked by simply launching the tool without any
240   * arguments.
241   *
242   * @return  {@code true} if this tool supports interactive mode, or
243   *          {@code false} if not.
244   */
245  @Override()
246  public boolean supportsInteractiveMode()
247  {
248    return true;
249  }
250
251
252
253  /**
254   * Indicates whether this tool defaults to launching in interactive mode if
255   * the tool is invoked without any command-line arguments.  This will only be
256   * used if {@link #supportsInteractiveMode()} returns {@code true}.
257   *
258   * @return  {@code true} if this tool defaults to using interactive mode if
259   *          launched without any command-line arguments, or {@code false} if
260   *          not.
261   */
262  @Override()
263  public boolean defaultsToInteractiveMode()
264  {
265    return true;
266  }
267
268
269
270  /**
271   * Indicates whether this tool supports the use of a properties file for
272   * specifying default values for arguments that aren't specified on the
273   * command line.
274   *
275   * @return  {@code true} if this tool supports the use of a properties file
276   *          for specifying default values for arguments that aren't specified
277   *          on the command line, or {@code false} if not.
278   */
279  @Override()
280  public boolean supportsPropertiesFile()
281  {
282    return true;
283  }
284
285
286
287  /**
288   * Indicates whether the LDAP-specific arguments should include alternate
289   * versions of all long identifiers that consist of multiple words so that
290   * they are available in both camelCase and dash-separated versions.
291   *
292   * @return  {@code true} if this tool should provide multiple versions of
293   *          long identifiers for LDAP-specific arguments, or {@code false} if
294   *          not.
295   */
296  @Override()
297  protected boolean includeAlternateLongIdentifiers()
298  {
299    return true;
300  }
301
302
303
304  /**
305   * Adds the arguments needed by this command-line tool to the provided
306   * argument parser which are not related to connecting or authenticating to
307   * the directory server.
308   *
309   * @param  parser  The argument parser to which the arguments should be added.
310   *
311   * @throws  ArgumentException  If a problem occurs while adding the arguments.
312   */
313  @Override()
314  public void addNonLDAPArguments(final ArgumentParser parser)
315         throws ArgumentException
316  {
317    String description = "The search base DN(s) to use to find entries with " +
318         "references to other entries.  At least one base DN must be " +
319         "specified.";
320    baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
321         description);
322    baseDNArgument.addLongIdentifier("base-dn");
323    parser.addArgument(baseDNArgument);
324
325    description = "The attribute(s) for which to find missing references.  " +
326         "At least one attribute must be specified, and each attribute " +
327         "must be indexed for equality searches and have values which are DNs.";
328    attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
329         description);
330    parser.addArgument(attributeArgument);
331
332    description = "The maximum number of entries to retrieve at a time when " +
333         "attempting to find entries with references to other entries.  This " +
334         "requires that the authenticated user have permission to use the " +
335         "simple paged results control, but it can avoid problems with the " +
336         "server sending entries too quickly for the client to handle.  By " +
337         "default, the simple paged results control will not be used.";
338    pageSizeArgument =
339         new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
340              description, 1, Integer.MAX_VALUE);
341    pageSizeArgument.addLongIdentifier("simple-page-size");
342    parser.addArgument(pageSizeArgument);
343  }
344
345
346
347  /**
348   * Performs the core set of processing for this tool.
349   *
350   * @return  A result code that indicates whether the processing completed
351   *          successfully.
352   */
353  @Override()
354  public ResultCode doToolProcessing()
355  {
356    // Establish a connection to the target directory server to use for
357    // finding references to entries.
358    final LDAPConnection findReferencesConnection;
359    try
360    {
361      findReferencesConnection = getConnection();
362    }
363    catch (final LDAPException le)
364    {
365      Debug.debugException(le);
366      err("Unable to establish a connection to the directory server:  ",
367           StaticUtils.getExceptionMessage(le));
368      return le.getResultCode();
369    }
370
371    try
372    {
373      // Establish a second connection to use for retrieving referenced entries.
374      try
375      {
376        getReferencedEntriesConnection = getConnection();
377      }
378      catch (final LDAPException le)
379      {
380        Debug.debugException(le);
381        err("Unable to establish a connection to the directory server:  ",
382             StaticUtils.getExceptionMessage(le));
383        return le.getResultCode();
384      }
385
386
387      // Get the set of attributes for which to find missing references.
388      final List<String> attrList = attributeArgument.getValues();
389      attributes = new String[attrList.size()];
390      attrList.toArray(attributes);
391
392
393      // Construct a search filter that will be used to find all entries with
394      // references to other entries.
395      final Filter filter;
396      if (attributes.length == 1)
397      {
398        filter = Filter.createPresenceFilter(attributes[0]);
399        missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
400      }
401      else
402      {
403        final Filter[] orComps = new Filter[attributes.length];
404        for (int i=0; i < attributes.length; i++)
405        {
406          orComps[i] = Filter.createPresenceFilter(attributes[i]);
407          missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
408        }
409        filter = Filter.createORFilter(orComps);
410      }
411
412
413      // Iterate across all of the search base DNs and perform searches to find
414      // missing references.
415      for (final DN baseDN : baseDNArgument.getValues())
416      {
417        ASN1OctetString cookie = null;
418        do
419        {
420          final SearchRequest searchRequest = new SearchRequest(this,
421               baseDN.toString(), SearchScope.SUB, filter, attributes);
422          if (pageSizeArgument.isPresent())
423          {
424            searchRequest.addControl(new SimplePagedResultsControl(
425                 pageSizeArgument.getValue(), cookie, false));
426          }
427
428          SearchResult searchResult;
429          try
430          {
431            searchResult = findReferencesConnection.search(searchRequest);
432          }
433          catch (final LDAPSearchException lse)
434          {
435            Debug.debugException(lse);
436            searchResult = lse.getSearchResult();
437          }
438
439          if (searchResult.getResultCode() != ResultCode.SUCCESS)
440          {
441            err("An error occurred while attempting to search for missing " +
442                 "references to entries below " + baseDN + ":  " +
443                 searchResult.getDiagnosticMessage());
444            return searchResult.getResultCode();
445          }
446
447          final SimplePagedResultsControl pagedResultsResponse;
448          try
449          {
450            pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
451          }
452          catch (final LDAPException le)
453          {
454            Debug.debugException(le);
455            err("An error occurred while attempting to decode a simple " +
456                 "paged results response control in the response to a " +
457                 "search for entries below " + baseDN + ":  " +
458                 StaticUtils.getExceptionMessage(le));
459            return le.getResultCode();
460          }
461
462          if (pagedResultsResponse != null)
463          {
464            if (pagedResultsResponse.moreResultsToReturn())
465            {
466              cookie = pagedResultsResponse.getCookie();
467            }
468            else
469            {
470              cookie = null;
471            }
472          }
473        }
474        while (cookie != null);
475      }
476
477
478      // See if there were any missing references found.
479      boolean missingReferenceFound = false;
480      for (final Map.Entry<String,AtomicLong> e :
481           missingReferenceCounts.entrySet())
482      {
483        final long numMissing = e.getValue().get();
484        if (numMissing > 0L)
485        {
486          if (! missingReferenceFound)
487          {
488            err();
489            missingReferenceFound = true;
490          }
491
492          err("Found " + numMissing + ' ' + e.getKey() +
493               " references to entries that do not exist.");
494        }
495      }
496
497      if (missingReferenceFound)
498      {
499        return ResultCode.CONSTRAINT_VIOLATION;
500      }
501      else
502      {
503        out("No references were found to entries that do not exist.");
504        return ResultCode.SUCCESS;
505      }
506    }
507    finally
508    {
509      findReferencesConnection.close();
510
511      if (getReferencedEntriesConnection != null)
512      {
513        getReferencedEntriesConnection.close();
514      }
515    }
516  }
517
518
519
520  /**
521   * Retrieves a map that correlates the number of missing references found by
522   * attribute type.
523   *
524   * @return  A map that correlates the number of missing references found by
525   *          attribute type.
526   */
527  public Map<String,AtomicLong> getMissingReferenceCounts()
528  {
529    return Collections.unmodifiableMap(missingReferenceCounts);
530  }
531
532
533
534  /**
535   * Retrieves a set of information that may be used to generate example usage
536   * information.  Each element in the returned map should consist of a map
537   * between an example set of arguments and a string that describes the
538   * behavior of the tool when invoked with that set of arguments.
539   *
540   * @return  A set of information that may be used to generate example usage
541   *          information.  It may be {@code null} or empty if no example usage
542   *          information is available.
543   */
544  @Override()
545  public LinkedHashMap<String[],String> getExampleUsages()
546  {
547    final LinkedHashMap<String[],String> exampleMap =
548         new LinkedHashMap<String[],String>(1);
549
550    final String[] args =
551    {
552      "--hostname", "server.example.com",
553      "--port", "389",
554      "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
555      "--bindPassword", "password",
556      "--baseDN", "dc=example,dc=com",
557      "--attribute", "member",
558      "--attribute", "uniqueMember",
559      "--simplePageSize", "100"
560    };
561    exampleMap.put(args,
562         "Identify all entries below dc=example,dc=com in which either the " +
563              "member or uniqueMember attribute references an entry that " +
564              "does not exist.");
565
566    return exampleMap;
567  }
568
569
570
571  /**
572   * Indicates that the provided search result entry has been returned by the
573   * server and may be processed by this search result listener.
574   *
575   * @param  searchEntry  The search result entry that has been returned by the
576   *                      server.
577   */
578  public void searchEntryReturned(final SearchResultEntry searchEntry)
579  {
580    try
581    {
582      // Find attributes which references to entries that do not exist.
583      for (final String attr : attributes)
584      {
585        final List<Attribute> attrList =
586             searchEntry.getAttributesWithOptions(attr, null);
587        for (final Attribute a : attrList)
588        {
589          for (final String value : a.getValues())
590          {
591            try
592            {
593              final SearchResultEntry e =
594                   getReferencedEntriesConnection.getEntry(value, "1.1");
595              if (e == null)
596              {
597                err("Entry '", searchEntry.getDN(), "' includes attribute ",
598                     a.getName(), " that references entry '", value,
599                     "' which does not exist.");
600                missingReferenceCounts.get(attr).incrementAndGet();
601              }
602            }
603            catch (final LDAPException le)
604            {
605              Debug.debugException(le);
606              err("An error occurred while attempting to determine whether " +
607                   "entry '" + value + "' referenced in attribute " +
608                   a.getName() + " of entry '" + searchEntry.getDN() +
609                   "' exists:  " + StaticUtils.getExceptionMessage(le));
610              missingReferenceCounts.get(attr).incrementAndGet();
611            }
612          }
613        }
614      }
615    }
616    finally
617    {
618      final long count = entriesExamined.incrementAndGet();
619      if ((count % 1000L) == 0L)
620      {
621        out(count, " entries examined");
622      }
623    }
624  }
625
626
627
628  /**
629   * Indicates that the provided search result reference has been returned by
630   * the server and may be processed by this search result listener.
631   *
632   * @param  searchReference  The search result reference that has been returned
633   *                          by the server.
634   */
635  public void searchReferenceReturned(
636                   final SearchResultReference searchReference)
637  {
638    // No implementation is required.  This tool will not follow referrals.
639  }
640}