Skip to content

ROOT-5073: Explore changing the on-file byte format to little endian #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zzxuanyuan wants to merge 18 commits into from
Closed

ROOT-5073: Explore changing the on-file byte format to little endian #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
fa6654d
Added flag buffBigEndian into frombuf() tobuf()
zzxuanyuan Nov 4, 2015
0704a6b
Change tobuf()/frombuf() functions
zzxuanyuan Nov 14, 2015
347bdd9
Change the BigEndian representations
zzxuanyuan Nov 18, 2015
202db5f
Only change TBuffer functions and add fBufBigEndian
zzxuanyuan Nov 20, 2015
8f9c143
Change {Is,Set}{Endianess} to {Is,Set}Buf{Endianess}
zzxuanyuan Apr 12, 2016
37bae27
Add debug info and set gROOT to little endian in MainEvent.cxx
zzxuanyuan Apr 12, 2016
7e3ca53
Added debug messages
zzxuanyuan Apr 16, 2016
4028c7e
Added fVersion debug
zzxuanyuan Apr 17, 2016
ec110fd
Add debug message in TFile::GetStreamerInfo
zzxuanyuan Apr 17, 2016
015cf7b
Fix tag out of range issue and TTree ReadVersion error
zzxuanyuan Apr 19, 2016
d5888ee
Add some debug code and change back the version reading back to big e…
zzxuanyuan Apr 19, 2016
c664e27
Added IsBufBigEndian() to indicate the endianness in tobuf(buf, cnt |…
zzxuanyuan Apr 25, 2016
0378251
Fix some segmentation fault in roottest cases and CheckByteCount error
zzxuanyuan Apr 27, 2016
8fd7774
Remove debug code
zzxuanyuan Apr 28, 2016
783a231
Fix format of if statement and fix compiling warning.
zzxuanyuan May 22, 2016
dd6e812
Add explanation of new arguments in constructors of TKey and TBuffer
zzxuanyuan May 22, 2016
c02dd1b
Use kFALSE and kTRUE for Bool_t type instead of 0 and 1
zzxuanyuan May 22, 2016
fa2f71f
Add code explanation which determine if there is byte count in buffer
zzxuanyuan May 22, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
367 changes: 347 additions & 20 deletions core/base/inc/Bytes.h
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 8 additions & 4 deletions core/base/inc/TBuffer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class TBuffer : public TObject {
protected:
typedef std::vector<TVirtualArray*> CacheList_t;

Bool_t fBufBigEndian; //Big endian
Bool_t fMode; //Read or write mode
Int_t fVersion; //Buffer format version
Int_t fBufSize; //Size of buffer
Expand All @@ -55,7 +56,7 @@ class TBuffer : public TObject {
CacheList_t fCacheStack; //Stack of pointers to the cache where to temporarily store the value of 'missing' data members

// Default ctor
TBuffer() : TObject(), fMode(0), fVersion(0), fBufSize(0), fBuffer(0),
TBuffer() : TObject(), fBufBigEndian(1), fMode(0), fVersion(0), fBufSize(0), fBuffer(0),
fBufCur(0), fBufMax(0), fParent(0), fReAllocFunc(0), fCacheStack(0,(TVirtualArray*)0) {}

// TBuffer objects cannot be copied or assigned
Expand All @@ -74,16 +75,19 @@ class TBuffer : public TObject {
enum { kCannotHandleMemberWiseStreaming = BIT(17)}; //if set TClonesArray should not use member wise streaming
enum { kInitialSize = 1024, kMinimalSize = 128 };

TBuffer(EMode mode);
TBuffer(EMode mode, Int_t bufsiz);
TBuffer(EMode mode, Int_t bufsiz, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = 0);
TBuffer(EMode mode, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TBuffer(EMode mode, Int_t bufsiz, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TBuffer(EMode mode, Int_t bufsiz, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = 0, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
virtual ~TBuffer();

Int_t GetBufferVersion() const { return fVersion; }
Bool_t IsReading() const { return (fMode & kWrite) == 0; }
Bool_t IsWriting() const { return (fMode & kWrite) != 0; }
void SetReadMode();
void SetWriteMode();
Bool_t IsBufBigEndian() const { return fBufBigEndian; }
void SetBufBigEndian() { fBufBigEndian = 1; }
void SetBufLittleEndian() { fBufBigEndian = 0; }
void SetBuffer(void *buf, UInt_t bufsiz = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = 0);
ReAllocCharFun_t GetReAllocFunc() const;
void SetReAllocFunc(ReAllocCharFun_t reallocfunc = 0);
Expand Down
5 changes: 5 additions & 0 deletions core/base/inc/TROOT.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ friend TROOT *ROOT::Internal::GetROOT2();
std::atomic<TApplication*> fApplication; //Pointer to current application
TInterpreter *fInterpreter; //Command interpreter
Bool_t fBatch; //True if session without graphics
Bool_t fBufBigEndian; //True if user define the data format as Big Endian
Bool_t fEditHistograms; //True if histograms can be edited with the mouse
Bool_t fFromPopUp; //True if command executed from a popup menu
Bool_t fMustClean; //True if object destructor scans canvases
Expand Down Expand Up @@ -264,6 +265,8 @@ friend TROOT *ROOT::Internal::GetROOT2();
void Idle(UInt_t idleTimeInSec, const char *command = 0);
Int_t IgnoreInclude(const char *fname, const char *expandedfname);
Bool_t IsBatch() const { return fBatch; }
Bool_t IsBufBigEndian() const { return fBufBigEndian; }
Bool_t IsBufLittleEndian() const { return !fBufBigEndian; }
Bool_t IsExecutingMacro() const { return fExecutingMacro; }
Bool_t IsFolder() const { return kTRUE; }
Bool_t IsInterrupted() const { return fInterrupt; }
Expand Down Expand Up @@ -297,6 +300,8 @@ friend TROOT *ROOT::Internal::GetROOT2();
void SaveContext();
void SetApplication(TApplication *app) { fApplication = app; }
void SetBatch(Bool_t batch = kTRUE) { fBatch = batch; }
void SetBufBigEndian() { fBufBigEndian = kTRUE; }
void SetBufLittleEndian() { fBufBigEndian = kFALSE; }
void SetCutClassName(const char *name = "TCutG");
void SetDefCanvasName(const char *name = "c1") { fDefCanvasName = name; }
void SetEditHistograms(Bool_t flag = kTRUE) { fEditHistograms = flag; }
Expand Down
58 changes: 47 additions & 11 deletions core/base/src/TBuffer.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Buffer base class used for serializing objects.
#include "TBuffer.h"
#include "TClass.h"
#include "TProcessID.h"
#include "TROOT.h"

const Int_t kExtraSpace = 8; // extra space at end of buffer (used for free block count)

Expand All @@ -34,13 +35,32 @@ static char *R__NoReAllocChar(char *, size_t, size_t)
/// Create an I/O buffer object. Mode should be either TBuffer::kRead or
/// TBuffer::kWrite. By default the I/O buffer has a size of
/// TBuffer::kInitialSize (1024) bytes.
///
/// \param[in] mode Read mode or write mode of this TBuffer
/// \param[in] def Determining whether this TKey is created with endianness defined by global gROOT or defined by buffBigEndian
/// \param[in] buffBigEndian Determining the endianness of this TKey's buffer only if def is kFALSE
///
/// TBuffer might store StreamerInfo which is always stored as big endian on TFile. Therefore, we should get rid of the effect
/// of gROOT->IsBufBigEndian() and always define its buffer as big endian. For more explanation, take a look at the function:
///
/// TKey::TKey(TDirectory* motherDir, const TKey &orig, UShort_t pidOffset, Bool_t def, Bool_t buffBigEndian);
///
/// It provides details of why we need 'def' and 'buffBigEndian' and how to define them.

TBuffer::TBuffer(EMode mode)
TBuffer::TBuffer(EMode mode, Bool_t def, Bool_t buffBigEndian)
Copy link
Copy Markdown
Member

@pcanal pcanal May 5, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

coding convention: the documentation (just before the function) must be extended explain the meaning of the new arguments. When updating a function's documentation consider upgrading it to use the doxygen format.

{
fBufSize = kInitialSize;
fMode = mode;
fVersion = 0;
fParent = 0;
if (def) {
fBufBigEndian = gROOT->IsBufBigEndian();
} else {
if (buffBigEndian)
fBufBigEndian = kTRUE;
else
fBufBigEndian = kFALSE;
}

SetBit(kIsOwner);

Expand All @@ -56,13 +76,21 @@ TBuffer::TBuffer(EMode mode)
/// Create an I/O buffer object. Mode should be either TBuffer::kRead or
/// TBuffer::kWrite.

TBuffer::TBuffer(EMode mode, Int_t bufsiz)
TBuffer::TBuffer(EMode mode, Int_t bufsiz, Bool_t def, Bool_t buffBigEndian)
{
if (bufsiz < kMinimalSize) bufsiz = kMinimalSize;
fBufSize = bufsiz;
fMode = mode;
fVersion = 0;
fParent = 0;
fBufSize = bufsiz;
fMode = mode;
fVersion = 0;
fParent = 0;
if (def) {
fBufBigEndian = gROOT->IsBufBigEndian();
} else {
if (buffBigEndian)
fBufBigEndian = kTRUE;
else
fBufBigEndian = kFALSE;
}

SetBit(kIsOwner);

Expand All @@ -85,12 +113,20 @@ TBuffer::TBuffer(EMode mode, Int_t bufsiz)
/// is provided, a Fatal error will be issued if the Buffer attempts to
/// expand.

TBuffer::TBuffer(EMode mode, Int_t bufsiz, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc)
TBuffer::TBuffer(EMode mode, Int_t bufsiz, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc, Bool_t def, Bool_t buffBigEndian)
{
fBufSize = bufsiz;
fMode = mode;
fVersion = 0;
fParent = 0;
fBufSize = bufsiz;
fMode = mode;
fVersion = 0;
fParent = 0;
if (def) {
fBufBigEndian = gROOT->IsBufBigEndian();
} else {
if (buffBigEndian)
fBufBigEndian = kTRUE;
else
fBufBigEndian = kFALSE;
}

SetBit(kIsOwner);

Expand Down
2 changes: 1 addition & 1 deletion core/base/src/TDirectory.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ static TBuffer* R__CreateBuffer()
if (creator == 0) {
R__LOCKGUARD2(gROOTMutex);
TClass *c = TClass::GetClass("TBufferFile");
TMethod *m = c->GetMethodWithPrototype("TBufferFile","TBuffer::EMode,Int_t",kFALSE,ROOT::kExactMatch);
TMethod *m = c->GetMethodWithPrototype("TBufferFile","TBuffer::EMode,Int_t,Bool_t,Bool_t",kFALSE,ROOT::kExactMatch);
creator = (tcling_callfunc_Wrapper_t)( m->InterfaceMethod() );
}
TBuffer::EMode mode = TBuffer::kWrite;
Expand Down
4 changes: 2 additions & 2 deletions core/base/src/TROOT.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ ClassImp(TROOT)
TROOT::TROOT() : TDirectory(),
fLineIsProcessing(0), fVersion(0), fVersionInt(0), fVersionCode(0),
fVersionDate(0), fVersionTime(0), fBuiltDate(0), fBuiltTime(0),
fTimer(0), fApplication(0), fInterpreter(0), fBatch(kTRUE), fEditHistograms(kTRUE),
fTimer(0), fApplication(0), fInterpreter(0), fBatch(kTRUE), fBufBigEndian(kTRUE), fEditHistograms(kTRUE),
fFromPopUp(kTRUE),fMustClean(kTRUE),fReadingObject(kFALSE),fForceStyle(kFALSE),
fInterrupt(kFALSE),fEscape(kFALSE),fExecutingMacro(kFALSE),fEditorMode(0),
fPrimitive(0),fSelectPad(0),fClasses(0),fTypes(0),fGlobals(0),fGlobalFunctions(0),
Expand Down Expand Up @@ -507,7 +507,7 @@ TROOT::TROOT() : TDirectory(),
TROOT::TROOT(const char *name, const char *title, VoidFuncPtr_t *initfunc)
: TDirectory(), fLineIsProcessing(0), fVersion(0), fVersionInt(0), fVersionCode(0),
fVersionDate(0), fVersionTime(0), fBuiltDate(0), fBuiltTime(0),
fTimer(0), fApplication(0), fInterpreter(0), fBatch(kTRUE), fEditHistograms(kTRUE),
fTimer(0), fApplication(0), fInterpreter(0), fBatch(kTRUE), fBufBigEndian(kTRUE), fEditHistograms(kTRUE),
fFromPopUp(kTRUE),fMustClean(kTRUE),fReadingObject(kFALSE),fForceStyle(kFALSE),
fInterrupt(kFALSE),fEscape(kFALSE),fExecutingMacro(kFALSE),fEditorMode(0),
fPrimitive(0),fSelectPad(0),fClasses(0),fTypes(0),fGlobals(0),fGlobalFunctions(0),
Expand Down
36 changes: 18 additions & 18 deletions io/io/inc/TBufferFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ class TBufferFile : public TBuffer {
enum { kTextBasedStreaming = BIT(18) }; //indicates if buffer used for XML/SQL object streaming
enum { kUser1 = BIT(21), kUser2 = BIT(22), kUser3 = BIT(23)}; //free for user

TBufferFile(TBuffer::EMode mode);
TBufferFile(TBuffer::EMode mode, Int_t bufsiz);
TBufferFile(TBuffer::EMode mode, Int_t bufsiz, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = 0);
TBufferFile(TBuffer::EMode mode, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TBufferFile(TBuffer::EMode mode, Int_t bufsiz, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TBufferFile(TBuffer::EMode mode, Int_t bufsiz, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = 0, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
virtual ~TBufferFile();

Int_t GetMapCount() const { return fMapCount; }
Expand Down Expand Up @@ -350,70 +350,70 @@ inline void TBufferFile::WriteUChar(UChar_t c)
inline void TBufferFile::WriteShort(Short_t h)
{
if (fBufCur + sizeof(Short_t) > fBufMax) AutoExpand(fBufSize+sizeof(Short_t));
tobuf(fBufCur, h);
tobuf(fBufCur, h, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteUShort(UShort_t h)
{
if (fBufCur + sizeof(UShort_t) > fBufMax) AutoExpand(fBufSize+sizeof(UShort_t));
tobuf(fBufCur, (Short_t)h);
tobuf(fBufCur, (Short_t)h, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteInt(Int_t i)
{
if (fBufCur + sizeof(Int_t) > fBufMax) AutoExpand(fBufSize+sizeof(Int_t));
tobuf(fBufCur, i);
tobuf(fBufCur, i, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteUInt(UInt_t i)
{
if (fBufCur + sizeof(UInt_t) > fBufMax) AutoExpand(fBufSize+sizeof(UInt_t));
tobuf(fBufCur, (Int_t)i);
tobuf(fBufCur, (Int_t)i, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteLong(Long_t l)
{
if (fBufCur + sizeof(Long_t) > fBufMax) AutoExpand(fBufSize+sizeof(Long_t));
tobuf(fBufCur, l);
tobuf(fBufCur, l, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteULong(ULong_t l)
{
if (fBufCur + sizeof(ULong_t) > fBufMax) AutoExpand(fBufSize+sizeof(ULong_t));
tobuf(fBufCur, (Long_t)l);
tobuf(fBufCur, (Long_t)l, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteLong64(Long64_t ll)
{
if (fBufCur + sizeof(Long64_t) > fBufMax) AutoExpand(fBufSize+sizeof(Long64_t));
tobuf(fBufCur, ll);
tobuf(fBufCur, ll, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteULong64(ULong64_t ll)
{
if (fBufCur + sizeof(ULong64_t) > fBufMax) AutoExpand(fBufSize+sizeof(ULong64_t));
tobuf(fBufCur, (Long64_t)ll);
tobuf(fBufCur, (Long64_t)ll, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteFloat(Float_t f)
{
if (fBufCur + sizeof(Float_t) > fBufMax) AutoExpand(fBufSize+sizeof(Float_t));
tobuf(fBufCur, f);
tobuf(fBufCur, f, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::WriteDouble(Double_t d)
{
if (fBufCur + sizeof(Double_t) > fBufMax) AutoExpand(fBufSize+sizeof(Double_t));
tobuf(fBufCur, d);
tobuf(fBufCur, d, IsBufBigEndian());
}

//______________________________________________________________________________
Expand Down Expand Up @@ -443,7 +443,7 @@ inline void TBufferFile::ReadUChar(UChar_t &c)
//______________________________________________________________________________
inline void TBufferFile::ReadShort(Short_t &h)
{
frombuf(fBufCur, &h);
frombuf(fBufCur, &h, IsBufBigEndian());
}

//______________________________________________________________________________
Expand All @@ -455,7 +455,7 @@ inline void TBufferFile::ReadUShort(UShort_t &h)
//______________________________________________________________________________
inline void TBufferFile::ReadInt(Int_t &i)
{
frombuf(fBufCur, &i);
frombuf(fBufCur, &i, IsBufBigEndian());
}

//______________________________________________________________________________
Expand All @@ -482,7 +482,7 @@ inline void TBufferFile::ReadULong(ULong_t &ll)
//______________________________________________________________________________
inline void TBufferFile::ReadLong64(Long64_t &ll)
{
frombuf(fBufCur, &ll);
frombuf(fBufCur, &ll, IsBufBigEndian());
}

//______________________________________________________________________________
Expand All @@ -494,13 +494,13 @@ inline void TBufferFile::ReadULong64(ULong64_t &ll)
//______________________________________________________________________________
inline void TBufferFile::ReadFloat(Float_t &f)
{
frombuf(fBufCur, &f);
frombuf(fBufCur, &f, IsBufBigEndian());
}

//______________________________________________________________________________
inline void TBufferFile::ReadDouble(Double_t &d)
{
frombuf(fBufCur, &d);
frombuf(fBufCur, &d, IsBufBigEndian());
}

//______________________________________________________________________________
Expand Down
8 changes: 4 additions & 4 deletions io/io/inc/TKey.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@ class TKey : public TNamed {
public:
TKey();
TKey(TDirectory* motherDir);
TKey(TDirectory* motherDir, const TKey &orig, UShort_t pidOffset);
TKey(TDirectory* motherDir, const TKey &orig, UShort_t pidOffset, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TKey(const char *name, const char *title, const TClass *cl, Int_t nbytes, TDirectory* motherDir);
TKey(const TString &name, const TString &title, const TClass *cl, Int_t nbytes, TDirectory* motherDir);
TKey(const TObject *obj, const char *name, Int_t bufsize, TDirectory* motherDir);
TKey(const void *obj, const TClass *cl, const char *name, Int_t bufsize, TDirectory* motherDir);
TKey(const TObject *obj, const char *name, Int_t bufsize, TDirectory* motherDir, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TKey(const void *obj, const TClass *cl, const char *name, Int_t bufsize, TDirectory* motherDir, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
TKey(Long64_t pointer, Int_t nbytes, TDirectory* motherDir = 0);
virtual ~TKey();

Expand Down Expand Up @@ -98,7 +98,7 @@ class TKey : public TNamed {
virtual void Print(Option_t *option="") const;
virtual Int_t Read(TObject *obj);
virtual TObject *ReadObj();
virtual TObject *ReadObjWithBuffer(char *bufferRead);
virtual TObject *ReadObjWithBuffer(char *bufferRead, Bool_t def = kTRUE, Bool_t buffBigEndian = kTRUE);
virtual void *ReadObjectAny(const TClass *expectedClass);
virtual void ReadBuffer(char *&buffer);
void ReadKeyBuffer(char *&buffer);
Expand Down
Loading