KKB::XmlTokenizer Class Reference

Manages the break down a stream into a set of logical tokens compatible with the XML format. More...

#include <XmlTokenizer.h>

Public Member Functions
	XmlTokenizer (TokenBufferPtr _in)
	Constructs a XmlTokenizer using the provided [[TokenBuffer]] _in as the data stream source. More...
	XmlTokenizer (const KKStr &_str)
	Manages the extraction of xml tokens from a KKStr instance; accomplishes this by building a [[TokenBufferStr]] around _str. More...
	XmlTokenizer (const KKStr &_fileName, bool &_fileOpened)
	~XmlTokenizer ()
bool	EndOfFile ()
KKStrPtr	GetNextToken ()
	Will retrieve the next token in the stream which will be either a tag token or up to one line of the content part of an element. If it is a content token it may end with a ' ' character. The idea is tat when reading content we will never return more than one line of text at a time. More...
KKStrListPtr	GetNextTokens (const KKStr &delToken)
	Returns a list of tokens up to and including the first occurrence of 'delToken'. More...
KKStrConstPtr	operator[] (kkuint32 idx)
KKStrConstPtr	Peek (kkuint32 idx)
	Allows you to look at future tokens in the stream; index of 0 would be the next token to be extracted. More...
void	PushTokenOnFront (KKStrPtr t)
	places token at current position such that it will be the next token extracted from the stream. More...

Detailed Description

Manages the break down a stream into a set of logical tokens compatible with the XML format.

Author

Kurt Kramer

Breaks up a source KKStr, text file, or [[TokenBuffer]] into logical tokens compatible with the XML format. XmlStream utilizes this object to parse streams. XmlStream

Definition at line 25 of file XmlTokenizer.h.

Constructor & Destructor Documentation

XmlTokenizer::XmlTokenizer

(

TokenBufferPtr

_in

)

Constructs a XmlTokenizer using the provided [[TokenBuffer]] _in as the data stream source.

Does NOT take ownership of _in; next token extracted will be from current position in _in.

Parameters

_in	Will retrieve tokens from starting from its current position; does not take ownership.

Definition at line 23 of file XmlTokenizer.cpp.

References XmlTokenizer().

Referenced by XmlTokenizer().

                                              :

  atEndOfFile      (false),
  in               (_in),
  tokenList        (),
  weOwnTokenBuffer (false)
  
#if  defined(_LogStream_)
  ,
  logger1          ("C:\\Temp\\XmlTokenizer-1.txt"),
  logger2          ("C:\\Temp\\XmlTokenizer-2.txt")
#endif
{
  Initialize ();
}

XmlTokenizer::XmlTokenizer

(

const KKStr &

_str

)

Manages the extraction of xml tokens from a KKStr instance; accomplishes this by building a [[TokenBufferStr]] around _str.

Definition at line 41 of file XmlTokenizer.cpp.

References KKB::TokenBufferStr::TokenBufferStr(), and XmlTokenizer().

Referenced by XmlTokenizer().

                                             :

  atEndOfFile       (false),
  in                (NULL),
  tokenList         (),
  weOwnTokenBuffer  (false)
#if  defined(_LogStream_)
  ,
  logger1           ("C:\\Temp\\XmlTokenizer-1.txt"),
  logger2           ("C:\\Temp\\XmlTokenizer-2.txt")
#endif
{
  in = new TokenBufferStr (_str);
  weOwnTokenBuffer = true;
  Initialize ();
}

XmlTokenizer::XmlTokenizer	(	const KKStr &	_fileName,
		bool &	_fileOpened
	)

Definition at line 60 of file XmlTokenizer.cpp.

References KKB::TokenBufferStream::TokenBufferStream(), KKB::TokenBuffer::Valid(), and XmlTokenizer().

Referenced by KKB::XmlStream::XmlStream(), and XmlTokenizer().

                            :

  atEndOfFile       (false),
  in                (NULL),
  tokenList         (),
  weOwnTokenBuffer  (false)
#if  defined(_LogStream_)
  ,
  logger1           ("C:\\Temp\\XmlTokenizer-1.txt"),
  logger2           ("C:\\Temp\\XmlTokenizer-2.txt")
#endif
{
  in = new TokenBufferStream (_fileName);
  _fileOpened = (in->Valid ());
  if  (_fileOpened)
  {
    weOwnTokenBuffer = true;
    Initialize ();
  }
}

XmlTokenizer::~XmlTokenizer

(

)

Definition at line 86 of file XmlTokenizer.cpp.

{
  if  (weOwnTokenBuffer)
  {
    delete  in;
    in = NULL;
  }
}

Member Function Documentation

bool XmlTokenizer::EndOfFile

(

)

Indicates if there anymore tokens that can be extracted.

Definition at line 224 of file XmlTokenizer.cpp.

{
//  if  (tokenList.QueueSize () == 0)
//    return true;
  while  ((tokenList.size () < 1)  &&  (!atEndOfFile))
    ReadInNextLogicalToken ();

  return  (tokenList.size () < 1);
}  /* EndOfFile */

KKStrPtr XmlTokenizer::GetNextToken

(

)

Will retrieve the next token in the stream which will be either a tag token or up to one line of the content part of an element. If it is a content token it may end with a '
' character. The idea is tat when reading content we will never return more than one line of text at a time.

Definition at line 135 of file XmlTokenizer.cpp.

Referenced by KKB::XmlStream::GetNextContent(), KKB::XmlStream::GetNextToken(), and GetNextTokens().

{
  while  ((tokenList.size () < 1)  && (!atEndOfFile))
    ReadInNextLogicalToken ();

  if  (tokenList.size () < 1)
  {
    #if  defined(_LogStream_)
    logger2 << "GetNextToken    return NULL" << endl;
    logger2.flush ();
    #endif
    return NULL;
  }

  kkuint32 s = tokenList.size ();

  KKStrPtr t = tokenList.front ();
  tokenList.pop_front ();

  #if  defined(_LogStream_)
  logger2 << "GetNextToken size[" << s << "] :" << (t ? (*t) : "NULL") << endl;
  logger2.flush ();
  #endif

  return  t;
}  /* GetNextToken */

KKStrListPtr XmlTokenizer::GetNextTokens

(

const KKStr &

delToken

)

Returns a list of tokens up to and including the first occurrence of 'delToken'.

Will return a list of tokens up to and including the first occurrence if 'delToken'.

Caller will take ownership of the returned tokens, and be responsible for deleting them.

Definition at line 167 of file XmlTokenizer.cpp.

References KKB::KKStr::Empty(), GetNextToken(), KKB::KKStrList::KKStrList(), and KKB::KKStr::operator!=().

{
  if  (delToken.Empty ())
    return NULL;

  if  (atEndOfFile  &&  (tokenList.size () < 1))
    return NULL;

  KKStrPtr  t = GetNextToken ();
  if  (t == NULL)
    return NULL;

  KKStrListPtr  tokens = new KKStrList (true);
  while  ((t != NULL)  &&  (*t != delToken))
  {
    tokens->PushOnBack (t);
    t = GetNextToken ();
  }

  if  (t)
    tokens->PushOnBack (t);

  return  tokens;
}  /* GetNextTokens */

KKStrConstPtr XmlTokenizer::operator[]

(

kkuint32

idx

)

Returns pointers to following Tokens in the stream where idx==0 indicates the next token.

Definition at line 481 of file XmlTokenizer.cpp.

References Peek().

{
  return Peek (idx);
}  /* operator[] */

KKStrConstPtr XmlTokenizer::Peek

(

kkuint32

idx

)

Allows you to look at future tokens in the stream; index of 0 would be the next token to be extracted.

Definition at line 202 of file XmlTokenizer.cpp.

Referenced by KKB::XmlStream::GetNextContent(), and operator[]().

{
  while  ((tokenList.size () < (idx + 1))  &&  !atEndOfFile)
    ReadInNextLogicalToken ();

  if  (idx >= tokenList.size ())
  {
    #if  defined(_LogStream_)
    logger2 << "Peek  idx[" << idx << "]   returning NULL" << endl;
    #endif
    return NULL;
  }

  #if  defined(_LogStream_)
  logger2 << "Peek  idx[" << idx << "]  :" << *(tokenList[idx]) << endl;
  #endif

  return  tokenList[idx];
}  /* Peek */

void XmlTokenizer::PushTokenOnFront

(

KKStrPtr

t

)

places token at current position such that it will be the next token extracted from the stream.

Definition at line 194 of file XmlTokenizer.cpp.

{
  tokenList.push_front (t);
}

---

The documentation for this class was generated from the following files:

• C:/Users/Kurt/GitHub/KSquareLibraries/KKBase/XmlTokenizer.h
• C:/Users/Kurt/GitHub/KSquareLibraries/KKBase/XmlTokenizer.cpp