Detects duplicate images in a given FeaureVectorList objects. More...

#include <DuplicateImages.h>

Public Member Functions
	DuplicateImages (FeatureVectorListPtr _examples, RunLog &_log)
	You would use this instance to search for duplicates in the list of 'examples'. More...

	DuplicateImages (FileDescPtr _fileDesc, RunLog &_log)

	~DuplicateImages ()

bool	AddExamples (FeatureVectorListPtr examples)
	Will add all the examples; be careful of ownership. More...

DuplicateImagePtr	AddSingleExample (FeatureVectorPtr example)
	Add one more FeatureVector to the list. More...

DuplicateImageListPtr	DupExamples () const

kkint32	DuplicateCount () const

kkint32	DuplicateDataCount () const

kkint32	DuplicateNameCount () const

bool	DuplicatesFound () const

bool	ExampleInDetector (FeatureVectorPtr fv)

FeatureVectorListPtr	ListOfExamplesToDelete ()

void	PurgeDuplicates (FeatureVectorListPtr examples, bool allowDupsInSameClass, std::ostream *report)
	Delete duplicate examples from FeatureVectorList structure provided in constructor. More...

void	ReportDuplicates (std::ostream &o)

Detailed Description

Detects duplicate images in a given FeaureVectorList objects.

Author: Kurt Kramer

Will derive a list of duplicate FeatureVector objects in a given list. It will use both the Image File Name and feature data to detect duplicates. A duplicate can be detected in two ways. If two or more entries have the same ExampleFileName or FeatureData.

The simplest way to use this object is to create an instance with a FeatureVectorList object that you are concerned with. Then call the method DupExamples (), which will return the list of duplicates found via a structure called DuplicateImageList.

Definition at line 53 of file DuplicateImages.h.

Constructor & Destructor Documentation

DuplicateImages::DuplicateImages	(	FeatureVectorListPtr	_examples,
		RunLog &	_log
	)

You would use this instance to search for duplicates in the list of 'examples'.

You can still call 'AddExamples' and 'AddSingleExample';

Definition at line 25 of file DuplicateImages.cpp.

References KKMLL::DuplicateImageList::DuplicateImageList(), KKMLL::FeatureVectorList::FileDesc(), KKMLL::ImageFeaturesDataIndexed::ImageFeaturesDataIndexed(), and KKMLL::ImageFeaturesNameIndexed::ImageFeaturesNameIndexed().

Referenced by KKMLL::FeatureVectorList::RemoveDuplicateEntries().

                                   :
    duplicateCount     (0),
    duplicateDataCount (0),
    duplicateNameCount (0),
    dupExamples        (new DuplicateImageList (true)),
    featureDataTree    (new ImageFeaturesDataIndexed ()),
    fileDesc           (NULL),
    log                (_log),
    nameTree           (new ImageFeaturesNameIndexed ())
 
 {
   if  (!_examples)
   {
     log.Level (-1) << endl << endl << "DuplicateImages::DuplicateImages   ***ERROR***   '_examples == NULL'" << endl << endl;
     return;
   }
   fileDesc = _examples->FileDesc ();
   FindDuplicates (_examples);
 }

DuplicateImages::DuplicateImages	(	FileDescPtr	_fileDesc,
		RunLog &	_log
	)

Definition at line 50 of file DuplicateImages.cpp.

References KKMLL::DuplicateImageList::DuplicateImageList(), KKMLL::ImageFeaturesDataIndexed::ImageFeaturesDataIndexed(), and KKMLL::ImageFeaturesNameIndexed::ImageFeaturesNameIndexed().

Referenced by KKMLL::TrainingProcess2::CreateTrainingProcessForLevel().

                                   :
    duplicateCount     (0),
    duplicateDataCount (0),
    duplicateNameCount (0),
    dupExamples        (new DuplicateImageList (true)),
    featureDataTree    (new ImageFeaturesDataIndexed ()),
    fileDesc           (_fileDesc),
    log                (_log),
    nameTree           (new ImageFeaturesNameIndexed ())
 
 {
 }

DuplicateImages::~DuplicateImages ( void )

Definition at line 68 of file DuplicateImages.cpp.

 {
   delete  nameTree;         nameTree        = NULL;
   delete  featureDataTree;  featureDataTree = NULL;
   delete  dupExamples;      dupExamples     = NULL;
 }

Member Function Documentation

bool DuplicateImages::AddExamples ( FeatureVectorListPtr examples )

Will add all the examples; be careful of ownership.

Definition at line 90 of file DuplicateImages.cpp.

 {
   bool  dupsDetected = false;
   FeatureVectorList::iterator idx;
   for (idx = examples->begin ();  idx != examples->end ();  idx++)
   {
     DuplicateImagePtr dupExample = AddSingleExample (*idx);
     if  (dupExample)
       dupsDetected = true;
   }
 
   return dupsDetected;
 }  /* AddExamples */

DuplicateImagePtr DuplicateImages::AddSingleExample ( FeatureVectorPtr example )

Add one more FeatureVector to the list.

Will add one more example to list and if it turns out to be a duplicate will return pointer to a "DuplicateImage" structure that will contain a list of all images that it is duplicate to. If no duplicate found will then return a NULL pointer.

Parameters

[in] example FeatureVecvtor that you want to add to the list.

Definition at line 111 of file DuplicateImages.cpp.

References KKMLL::DuplicateImage::AddADuplicate(), KKB::KKStr::Concat(), KKMLL::DuplicateImage::DuplicateImage(), KKB::KKStr::Empty(), KKMLL::FeatureVector::ExampleFileName(), KKMLL::ImageFeaturesDataIndexed::GetEqual(), KKMLL::DuplicateImageList::LocateByImage(), and KKMLL::ImageFeaturesDataIndexed::RBInsert().

 {
   DuplicateImagePtr dupExample = NULL;
 
   FeatureVectorPtr    existingNameExample = NULL;
 
   const KKStr&  imageFileName = example->ExampleFileName ();
   if  (!imageFileName.Empty ())
   {
     existingNameExample = nameTree->GetEqual (osGetRootName (example->ExampleFileName ()));
     if  (!existingNameExample)
       nameTree->RBInsert (example);
   }
 
   FeatureVectorPtr  existingDataExample = featureDataTree->GetEqual (example);
   if  (!existingDataExample)
     featureDataTree->RBInsert (example);
   
   if  ((existingNameExample)  ||  (existingDataExample))
   {
     duplicateCount++;
     if  (existingNameExample)
     {
       duplicateNameCount++;
       dupExample = dupExamples->LocateByImage (existingNameExample);
       if  (!dupExample)
       {
         dupExample = new DuplicateImage (fileDesc, existingNameExample, example, log);
         dupExamples->PushOnBack (dupExample);
       }
       else
       {
         dupExample->AddADuplicate (example);
       }
     }
     
     if  (existingDataExample) 
     {
       duplicateDataCount++;
       if  (existingDataExample != existingNameExample)
       {
         dupExample = dupExamples->LocateByImage (existingDataExample);
         if  (!dupExample)
         {
           dupExample = new DuplicateImage (fileDesc, existingDataExample, example, log);
           dupExamples->PushOnBack (dupExample);
         }
         else
         {
           dupExample->AddADuplicate (example);
         }
       }
     }
   }
 
   return dupExample;
 }  /* AddSingleExample */

DuplicateImageListPtr KKMLL::DuplicateImages::DupExamples ( ) const

inline

Definition at line 85 of file DuplicateImages.h.

Referenced by ListOfExamplesToDelete(), and PurgeDuplicates().

85 {return dupExamples;}

kkint32 KKMLL::DuplicateImages::DuplicateCount ( ) const

inline

Definition at line 87 of file DuplicateImages.h.

87 {return duplicateCount;}

kkint32 KKMLL::DuplicateImages::DuplicateDataCount ( ) const

inline

Definition at line 88 of file DuplicateImages.h.

88 {return duplicateDataCount;}

kkint32 KKMLL::DuplicateImages::DuplicateNameCount ( ) const

inline

Definition at line 89 of file DuplicateImages.h.

89 {return duplicateNameCount;}

bool DuplicateImages::DuplicatesFound ( ) const

Definition at line 362 of file DuplicateImages.cpp.

 {
   return  (dupExamples->QueueSize () > 0);
 }

bool DuplicateImages::ExampleInDetector ( FeatureVectorPtr fv )

Definition at line 77 of file DuplicateImages.cpp.

References KKMLL::ImageFeaturesDataIndexed::GetEqual().

 {
   if  (nameTree->GetEqual (fv->ExampleFileName ()) != NULL)
     return true;
 
   if  (featureDataTree->GetEqual (fv) != NULL)
     return true;
 
   return false;
 }  /* ExampleInDetector */

FeatureVectorListPtr DuplicateImages::ListOfExamplesToDelete ( )

Definition at line 276 of file DuplicateImages.cpp.

References KKMLL::DuplicateImage::AllTheSameClass(), DupExamples(), KKMLL::DuplicateImage::DuplicatedImages(), KKMLL::DuplicateImage::ExampleWithSmallestScanLine(), KKMLL::FeatureVectorList::FeatureVectorList(), and KKMLL::FeatureVectorList::PushOnBack().

 {
   FeatureVectorListPtr  examplesToDelete = new FeatureVectorList (fileDesc, false);
 
   log.Level (10) << "DuplicateImages::ListOfExamplesToDelete" << endl;
 
   DuplicateImageListPtr  dupExamples = DupExamples ();
 
   DuplicateImageList::iterator  dIDX = dupExamples->begin ();
 
   for  (dIDX = dupExamples->begin ();   dIDX != dupExamples->end ();  ++dIDX)
   {
     DuplicateImagePtr dupSet = *dIDX;
 
     log.Level (20) << "ListOfExamplesToDelete   Duplicate Set[" << dupSet->FirstExampleAdded ()->ExampleFileName () << "]" << endl;
 
     FeatureVectorListPtr  examplesInSet = dupSet->DuplicatedImages ();
     FeatureVectorPtr exampleToKeep = NULL;
 
     if  (dupSet->AllTheSameClass ())
     {
       exampleToKeep = dupSet->ExampleWithSmallestScanLine ();
     }
     
     FeatureVectorList::iterator iIDX = examplesInSet->begin ();
 
     for  (iIDX = examplesInSet->begin ();  iIDX != examplesInSet->end ();  ++iIDX)
     {
       FeatureVectorPtr example = *iIDX;
       if  (!example)
         continue;
 
       if  (example == exampleToKeep)
       {
         log.Level (30) << "ListOfExamplesToDelete  Keeping [" << exampleToKeep->ExampleFileName () << "]." << endl;
       }
       else
       {
         log.Level (30) << "ListOfExamplesToDelete  Deleting [" << example->ExampleFileName () << "]." << endl;
         examplesToDelete->PushOnBack (example);
       }
     }
   }
 
   return  examplesToDelete;
 }  /* ListOfExamplesToDelete */

void DuplicateImages::PurgeDuplicates	(	FeatureVectorListPtr	examples,
		bool	allowDupsInSameClass,
		std::ostream *	report
	)

Delete duplicate examples from FeatureVectorList structure provided in constructor.

if not equal NULL will list examples being purged.

If duplicates are in more than one class then all will be deleted. if duplicates are in a single class then one with smallest scan line will be kept while all others will be deleted.

Definition at line 194 of file DuplicateImages.cpp.

References KKMLL::DuplicateImage::AllTheSameClass(), DupExamples(), KKMLL::DuplicateImage::DuplicatedImages(), and KKMLL::DuplicateImage::ExampleWithSmallestScanLine().

Referenced by KKMLL::TrainingProcess2::CreateTrainingProcessForLevel(), and KKMLL::FeatureVectorList::RemoveDuplicateEntries().

 {
   log.Level (10) << "DuplicateImageList::PurgeDuplicates" << endl;
 
   
   // To make sure that we do not delete the same example Twice I added 'deletedDictionary' below.
   // if will track all examples by address that have been deleted.  I did this because a bug in
   // the duplicate detector routine had the same example added to to different groups of duplicates.
   map<FeatureVectorPtr,KKStr>  deletedDictionary;  // List of examples already deleted.
   map<FeatureVectorPtr,KKStr>::iterator  deletedDictionaryIdx;
   
   DuplicateImageListPtr  dupExamples = DupExamples ();
 
   kkint32  dupSetCount = 0;  
   DuplicateImageList::iterator  dIDX = dupExamples->begin ();
 
   for  (dIDX = dupExamples->begin ();   dIDX != dupExamples->end ();  ++dIDX, ++dupSetCount)
   {
     DuplicateImagePtr dupSet = *dIDX;
 
     log.Level (20) << "PurgeDuplicates  Duplicate Set[" << dupSet->FirstExampleAdded ()->ExampleFileName () << "]" << endl;
 
     FeatureVectorListPtr  examplesInSet = dupSet->DuplicatedImages ();
     FeatureVectorPtr exampleToKeep = NULL;
 
     if  (dupSet->AllTheSameClass ())
     {
       if  (allowDupsInSameClass)
         continue;
       else
         exampleToKeep = dupSet->ExampleWithSmallestScanLine ();
     }
     
     FeatureVectorList::iterator iIDX = examplesInSet->begin ();
 
     for  (iIDX = examplesInSet->begin ();  iIDX != examplesInSet->end ();  ++iIDX)
     {
       FeatureVectorPtr example = *iIDX;
       if  (!example)
         continue;
 
       if  (example == exampleToKeep)
       {
         log.Level (30) << "PurgeDuplicates  Keeping [" << exampleToKeep->ExampleFileName () << "]." << endl;
         if  (report)
           *report << example->ExampleFileName () << "\t" << "Class" << "\t" << example->MLClassName () << "\t" << "Duplicate retained." << endl;
       }
       else
       {
         bool  alreadyDeleted = false;
         deletedDictionaryIdx = deletedDictionary.find (example);
         if  (deletedDictionaryIdx != deletedDictionary.end ())
         {
           // AHA We are getting ready to delete an entry we have already deleted ????
           KKStr  errMsg (1024);
           errMsg << "Example: " << deletedDictionaryIdx->second << "  Already Been Deleted.";
           log.Level (-1) << endl << "DuplicateImages::PurgeDuplicates   ***ERROR***  " << errMsg << endl <<endl;
           alreadyDeleted = true;
         }
 
         if  (!alreadyDeleted)
         {
           deletedDictionary.insert (pair<FeatureVectorPtr,KKStr> (example, example->ExampleFileName ()));
 
           log.Level (30) << "PurgeDuplicates  Deleting [" << example->ExampleFileName () << "]." << endl;
           if  (report)
             *report << example->ExampleFileName () << "\t" << "Class" << "\t" << example->MLClassName () << "\t" << "Duplicate deleted." << endl;
           examples->DeleteEntry (example);
           if  (examples->Owner ())
             delete  example;
         }
       }
     }
   }
 }  /* PurgeDuplicates */

void DuplicateImages::ReportDuplicates ( std::ostream & o )

Definition at line 326 of file DuplicateImages.cpp.

 {
   o << "Number of Duplicate Groups [" << dupExamples->QueueSize () << "]" << endl;
   kkint32  groupNum = 0;
 
   //for  (DuplicateImageList::iterator  idx = dupExamples->begin ();  idx != dupExamples->end ();  idx++)
   for  (auto  dupExampleSet:  *dupExamples)
   {
     const FeatureVectorListPtr  dupList = dupExampleSet->DuplicatedImages ();
 
     o << "Group[" << groupNum << "] Contains [" << dupList->QueueSize () << "] Duplicates." << endl;
 
     kkint32  numOnLine = 0;
     //FeatureVectorList::const_iterator  fvIDX;
     for  (auto fvIDX: *dupList)   //  = dupList->begin ();  fvIDX != dupList->end ();  ++fvIDX)
     {
       if  (numOnLine > 8)
       {
         o << endl;
         numOnLine = 0;
       }
 
       if  (numOnLine > 0)
         o << "\t";
       o << fvIDX->ExampleFileName () << "[" << fvIDX->MLClassName () << "]";
 
       numOnLine++;
     }
     o << endl << endl;;
 
     groupNum++;
   }
 }  /* ReportDuplicates */

The documentation for this class was generated from the following files:

C:/Users/Kurt/GitHub/KSquareLibraries/KKMachineLearning/DuplicateImages.h
C:/Users/Kurt/GitHub/KSquareLibraries/KKMachineLearning/DuplicateImages.cpp

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation