KKB::Tokenizer Class Reference

Class is meant to break down a stream into a set of logical tokens. More...

#include <Tokenizer.h>

Public Member Functions
	Tokenizer (TokenBufferPtr _in)
	Tokenizer (const KKStr &_str)
	Tokenizer (const KKStr &_fileName, bool &_fileOpened)
	~Tokenizer ()
void	DefineOperatorChars (char *const _operatorChars)
bool	EndOfFile ()
KKStrPtr	GetNextToken ()
KKStrListPtr	GetNextTokens (const KKStr &delToken)
	Returns a list of tokens up to and including the first occurrence of 'delToken'. More...
KKStrConstPtr	operator[] (kkuint32 idx)
KKStrConstPtr	Peek (kkuint32 idx)
void	PushTokenOnFront (KKStrPtr t)

Detailed Description

Class is meant to break down a stream into a set of logical tokens.

Author

Kurt Kramer

This class was originally created while taking Non Linear Systems. It breaks up a source KKStr or text file into logical tokens. You can create your own source of characters by creating a Class derived from KKB::TokenBuffer.

Definition at line 23 of file Tokenizer.h.

Constructor & Destructor Documentation

Tokenizer::Tokenizer

(

TokenBufferPtr

_in

)

Definition at line 22 of file Tokenizer.cpp.

References Tokenizer().

Referenced by Tokenizer().

                                        :

  atEndOfFile           (false),
  in                    (_in),
  secondCharAtEndOfFile (false),
  operatorChars         (NULL),
  tokenList             (true),
  weOwnTokenBuffer      (false)
{
  Initialize ();
}

Tokenizer::Tokenizer

(

const KKStr &

_str

)

Definition at line 36 of file Tokenizer.cpp.

References KKB::TokenBufferStr::TokenBufferStr(), and Tokenizer().

Referenced by Tokenizer().

                                       :

  atEndOfFile           (false),
  in                    (NULL),
  secondCharAtEndOfFile (false),
  operatorChars         (NULL),
  tokenList             (true),
  weOwnTokenBuffer      (false)
{
  in = new TokenBufferStr (_str);
  weOwnTokenBuffer = true;
  Initialize ();
}

Tokenizer::Tokenizer	(	const KKStr &	_fileName,
		bool &	_fileOpened
	)

Definition at line 52 of file Tokenizer.cpp.

References KKB::TokenBufferStream::TokenBufferStream(), Tokenizer(), and KKB::TokenBuffer::Valid().

Referenced by Tokenizer().

                      :

  atEndOfFile           (false),
  in                    (NULL),
  secondCharAtEndOfFile (false),
  operatorChars         (NULL),
  tokenList             (true),
  weOwnTokenBuffer      (false)
{
  in = new TokenBufferStream (_fileName);
  _fileOpened = (in->Valid ());
  if  (_fileOpened)
  {
    weOwnTokenBuffer = true;
    Initialize ();
  }
}

Tokenizer::~Tokenizer

(

)

Definition at line 75 of file Tokenizer.cpp.

{
  if  (weOwnTokenBuffer)
  {
    delete  in;
    in = NULL;
  }
  delete  operatorChars;
  operatorChars = NULL;
}

Member Function Documentation

void Tokenizer::DefineOperatorChars

(

char *const

_operatorChars

)

Definition at line 109 of file Tokenizer.cpp.

References KKB::STRDUP().

{
  delete  operatorChars ;
  operatorChars = KKB::STRDUP (_operatorChars);
}

bool Tokenizer::EndOfFile

(

)

Definition at line 175 of file Tokenizer.cpp.

{
//  if  (tokenList.QueueSize () == 0)
//    return true;
  while  ((tokenList.QueueSize () < 1)  &&  (!atEndOfFile))
    ReadInNextLogicalToken ();

  return  (tokenList.QueueSize () < 1);
}  /* EndOfFile */

KKStrPtr Tokenizer::GetNextToken

(

)

Definition at line 116 of file Tokenizer.cpp.

Referenced by GetNextTokens().

{
  while  (tokenList.QueueSize () < 1)
    ReadInNextLogicalToken ();

  KKStrPtr t = tokenList.PopFromFront ();
  return  t;
}  /* GetNextToken */

KKStrListPtr Tokenizer::GetNextTokens

(

const KKStr &

delToken

)

Returns a list of tokens up to and including the first occurrence of 'delToken'.

Will return a list of tokens up to and including the first occurrence if 'delToken'.

Caller will take ownership of the returned tokens, and be responsible for deleting them.

Definition at line 130 of file Tokenizer.cpp.

References KKB::KKStr::Empty(), GetNextToken(), KKB::KKStrList::KKStrList(), and KKB::KKStr::operator!=().

{
  if  (delToken.Empty ())
    return NULL;

  KKStrListPtr  tokens = new KKStrList (true);
  KKStrPtr  t = GetNextToken ();
  if  (t == NULL)
    return NULL;

  while  ((t != NULL)  &&  (*t != delToken))
  {
    tokens->PushOnBack (t);
    t = GetNextToken ();
  }

  if  (t)
    tokens->PushOnBack (t);

  return  tokens;
}  /* GetNextTokens */

KKStrConstPtr Tokenizer::operator[]

(

kkuint32

idx

)

Returns pointers to following Tokens in the stream where idx==0 indicates the next token.

Definition at line 419 of file Tokenizer.cpp.

References Peek().

{
  return Peek (idx);
}  /* operator[] */

KKStrConstPtr Tokenizer::Peek

(

kkuint32

idx

)

Definition at line 162 of file Tokenizer.cpp.

Referenced by operator[]().

{
  while  ((tokenList.QueueSize () < (kkint32)(idx + 1))  &&  !atEndOfFile)
    ReadInNextLogicalToken ();

  if  (idx >= tokenList.size ())
    return NULL;

  return  tokenList.IdxToPtr ((kkint32)idx);
}  /* Peek */

void Tokenizer::PushTokenOnFront

(

KKStrPtr

t

)

Definition at line 154 of file Tokenizer.cpp.

{
  tokenList.PushOnFront (t);
}

---

The documentation for this class was generated from the following files:

• C:/Users/Kurt/GitHub/KSquareLibraries/KKBase/Tokenizer.h
• C:/Users/Kurt/GitHub/KSquareLibraries/KKBase/Tokenizer.cpp