# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19ca54-6db4-11ee-b3b4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-0", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "High-Performance Neural Networks\nfor Visual Object Classi\fcation\nDan C. Cire\u0018 san, Ueli Meier, Jonathan Masci,\nLuca M. Gambardella and J\u007f urgen Schmidhuber\nTechnical Report No. IDSIA-01-11\nJanuary 2011\nIDSIA / USI-SUPSI\nDalle Molle Institute for Arti\fcial Intelligence\nGalleria 2, 6928 Manno, Switzerland\nIDSIA is a joint institute of both University of Lugano (USI) and University of Applied Sciences of Southern Switzerland (SUPSI),\nand was founded in 1988 by the Dalle Molle Foundation which promoted quality of life.\nThis work was partially supported by the Swiss Commission for Technology and Innovation (CTI), Project n. 9688.1 IFF:\nIntelligent Fill in Form.arXiv:1102.0183v1  [cs.AI]  1 Feb 2011\nTechnical Report No. IDSIA-01-11 1\nHigh-Performance Neural Networks\nfor Visual Object Classi\fcation\nDan C. Cire\u0018 san, Ueli Meier, Jonathan Masci,\nLuca M. Gambardella and J\u007f urgen Schmidhuber\nJanuary 2011\nAbstract\nWe present a fast, fully parameterizable GPU implementation of Convolutional Neural\nNetwork variants. Our feature extractors are neither carefully designed nor pre-wired, but"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f161-6db4-11ee-a97d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-1", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "January 2011\nAbstract\nWe present a fast, fully parameterizable GPU implementation of Convolutional Neural\nNetwork variants. Our feature extractors are neither carefully designed nor pre-wired, but\nrather learned in a supervised way. Our deep hierarchical architectures achieve the best\npublished results on benchmarks for object classi\fcation (NORB, CIFAR10) and handwritten\ndigit recognition (MNIST), with error rates of 2.53%, 19.51%, 0.35%, respectively. Deep\nnets trained by simple back-propagation perform better than more shallow ones. Learning\nis surprisingly rapid. NORB is completely trained within \fve epochs. Test error rates on\nMNIST drop to 2.42%, 0.97% and 0.48% after 1, 3 and 17 epochs, respectively.\n1 Introduction\nThe human visual system e\u000eciently recognizes and localizes objects within cluttered scenes. For\narti\fcial systems, however, this is still di\u000ecult, due to viewpoint-dependent object variability,\nand the high in-class variability of many object types. Deep hierarchical neural models roughly\nmimick the nature of mammalian visual cortex, and by community consensus are among the most\npromising architectures for such tasks. The most successful hierarchical object recognition systems\nall extract localized features from input images, convolving image patches with \flters. Filter"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f162-6db4-11ee-89b5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-2", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "promising architectures for such tasks. The most successful hierarchical object recognition systems\nall extract localized features from input images, convolving image patches with \flters. Filter\nresponses are then repeatedly sub-sampled and re-\fltered, resulting in a deep feed-forward network\narchitecture whose output feature vectors are eventually classi\fed. One of the \frst hierarchical\nneural systems was the Neocognitron (Fukushima, 1980) which inspired many of the more recent\nvariants.\nUnsupervised learning methods applied to patches of natural images tend to produce localized\n\flters that resemble o\u000b-center-on-surround \flters, orientation-sensitive bar detectors, Gabor \flters\n(Schmidhuber et al. , 1996; Olshausen and Field, 1997; Hoyer and Hyv\u007f arinen, 2000). These \fndings\nin conjunction with experimental studies of the visual cortex justify the use of such \flters in the\nso-called standard model for object recognition (Riesenhuber and Poggio, 1999; Serre et al. , 2007;\nMutch and Lowe, 2008), whose \flters are \fxed, in contrast to those of Convolutional Neural"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f163-6db4-11ee-bb81-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-3", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Mutch and Lowe, 2008), whose \flters are \fxed, in contrast to those of Convolutional Neural\nNetworks (CNNs) (LeCun et al. , 1998; Behnke, 2003; Simard et al. , 2003), whose weights (\flters)\nare randomly initialized and changed in a supervised way using back-propagation (BP).\nDespite the hardware progress of the past decades, computational speed is still a limiting\nfactor for CNN architectures characterized by many building blocks typically set by trial and\nerror. To systematically test the impact of various architectures on classi\fcation performance,\nwe present a fast CNN implementation on Graphics Processing Units (GPUs). Previous GPU\nimplementations of CNNs (Chellapilla et al. , 2006; Uetz and Behnke, 2009) were hard-coded to\nsatisfy GPU hardware constraints, whereas our implementation is \rexible and fully online (i.e.,\nTechnical Report No. IDSIA-01-11 2\nweight updates after each image). It allows for training large CNNs within days instead of months,\nsuch that we can investigate the in\ruence of various structural parameters by exploring large\nparameter spaces (Pinto et al. , 2009) and performing error analysis on repeated experiments.\nWe evaluate various networks on the handwritten digit benchmark MNIST (LeCun et al. , 1998)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f164-6db4-11ee-893f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-4", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "We evaluate various networks on the handwritten digit benchmark MNIST (LeCun et al. , 1998)\nand two image classi\fcation benchmarks: NORB (LeCun et al. , 2004) and CIFAR10 (Krizhevsky,\n2009).\n2 Convolutional neural networks\nCNNs are hierarchical neural networks whose convolutional layers alternate with subsampling\nlayers, reminiscent of simple and complex cells in the primary visual cortex (Wiesel and Hubel,\n1959). CNNs vary in how convolutional and subsampling layers are realized and how the nets\nare trained. The CNN architecture considered in this study di\u000bers from the one of Simard et al.\n(2003) in the sense that after each CNN-layer an optional max-pooling layer (Scherer et al. , 2010)\ncan be used. Here we give a complete description of this independent implementation (Fig. 1).\n2.1 Image processing layer\nThe image processing layer is an optional pre-processing layer of prede\fned \flters that are kept\n\fxed during training. Thus additional information besides the raw input image can be provided\nto the network, such as edges and gradients. In particular, we \fnd that a contrast-extracting layer\n(Fukushima, 2003) helps to improve the recognition rate for NORB.\n2.2 Convolutional layer"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f165-6db4-11ee-9394-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-5", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "(Fukushima, 2003) helps to improve the recognition rate for NORB.\n2.2 Convolutional layer\nA convolutional layer is parametrized by the size and the number of the maps, kernel sizes, skipping\nfactors, and the connection table. Each layer has Mmaps of equal size ( Mx,My). A kernel (blue\nrectangle in Fig 1) of size ( Kx,Ky) is shifted over the valid region of the input image (i.e. the\nkernel has to be completely inside the image). The skipping factors SxandSyde\fne how many\npixels the \flter/kernel skips in x- and y-direction between subsequent convolutions. The size of\nthe output map is then de\fned as:\nMn\nx=Mn\u00001\nx\u0000Kn\nx\nSnx+ 1+ 1;Mn\ny=Mn\u00001\ny\u0000Kn\ny\nSny+ 1+ 1 (1)\nwhere index nindicates the layer. Each map in layer Lnis connected to at most Mn\u00001maps in\nlayerLn\u00001. Neurons of a given map share their weights but have di\u000berent receptive \felds.\n2.3 Max-pooling layer\nThe biggest architectural di\u000berence between our implementation and the CNN of LeCun et al."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f166-6db4-11ee-a291-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-6", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "2.3 Max-pooling layer\nThe biggest architectural di\u000berence between our implementation and the CNN of LeCun et al.\n(1998) is the use of a max-pooling layer instead of a sub-sampling layer. No such layer is used\nby Simard et al. (2003) who simply skips nearby pixels prior to convolution, instead of pooling\nor averaging. Scherer et al. (2010) found that max-pooling can lead to faster convergence, select\nsuperior invariant features, and improve generalization. The output of the max-pooling layer is\ngiven by the maximum activation over non-overlapping rectangular regions of size ( Kx,Ky). Maxpooling enables position invariance over larger local regions and downsamples the input image by\na factor ofKxandKyalong each direction.\nTechnical Report No. IDSIA-01-11 3\n2.4 Classi\fcation layer\nKernel sizes of convolutional \flters and max-pooling rectangles as well as skipping factors are\nchosen such that either the output maps of the last convolutional layer are downsampled to 1\npixel per map, or a fully connected layer combines the outputs of the topmost convolutional layer\ninto a 1D feature vector. The top layer is always fully connected, with one output unit per class\nlabel."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f167-6db4-11ee-8375-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-7", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "into a 1D feature vector. The top layer is always fully connected, with one output unit per class\nlabel.\nFigure 1: Architecture of a convolutional neural network. In this case, the convolutional layers\nare fully connected. Both convolutional layers use a kernel of 5 x 5 and skipping factors of 1.\n3 GPU implementation\nThe latest generation of NVIDIA GPUs, the 400 and 500 series (we use GTX 480 & GTX 580), has\nmany advantages over older GPUs, most notably the presence of a R/W L2 global cache for device\nmemory. This permits faster programs and simpli\fes writing the code. In fact, the corresponding\ntransfer of complexity into hardware alleviates many software and optimization problems. Our\nexperiments show that the CNN program becomes 2-3 times faster just by switching from GTX\n285 to GTX 480.\nTechnical Report No. IDSIA-01-11 4\nManual optimization of CUDA code is very time-consuming and error prone. We optimize for\nthe new architecture, relying on the L2 cache for many of the device memory accesses, instead of\nmanually writing code that uses textures and shared memory. Code obtained by this pragmatic\nstrategy is fast enough. We use the following types of optimization: pre-computed expressions,\nunrolled loops within template kernels, strided matrices to obtain coalesced memory accesses and"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f168-6db4-11ee-8f2d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-8", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "strategy is fast enough. We use the following types of optimization: pre-computed expressions,\nunrolled loops within template kernels, strided matrices to obtain coalesced memory accesses and\nregisters wherever possible. Additional manual optimizations are possible in case future image\nclassi\fcation problems will require even more computing power.\n3.1 Data structures\nBoth outputs yand deltas\u000eof layerLnare 2D strided. Their original size is Mx\u0002MM y, but they\nare horizontally strided with a pitch of 32 \roats (we use this stride for all 2D data), resulting in\ncoalesced memory accesses. The vertical stride avoids additional bounding tests in CUDA kernels.\nAll connections between maps of consecutive layers Ln\u00001andLnare stored in matrix Cn.\nEach row of Cncontains all connections that feed into a particular map in layer Ln. Because\nwe aim for a \rexible architecture with partially connected layers, in the \frst column we store the\nnumber of previous connections. This index is useful for Forward Propagation (FP) and Adjusting\nWeights (AW) CUDA kernels. The second column stores the number of connections, followed by\ncorresponding indices of maps in Ln\u00001connected to the current map."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f169-6db4-11ee-b8a8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-9", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Weights (AW) CUDA kernels. The second column stores the number of connections, followed by\ncorresponding indices of maps in Ln\u00001connected to the current map.\nFor BP and FP, analogous information about connections is needed. We therefore store backward connections in CBP. AW requires a list of all map connections (see Subsection 3.4), stored\nas an array of map index pairs. Dealing with biases in BP kernel requires to know where the\nweights of particular connections start; this information is stored in a 2D array WIDX BPof size\nMn\u0002Mn\u00001.\n3.2 Forward propagation\nA straightforward way of parallelizing FP is to assign a thread block to each map that has to\nbe computed. For maps bigger than 1024 neurons, the job is further split into smaller blocks by\nassigning a block to each line of the map, because the number of threads per block is limited (1024\nfor GTX 480). A one to one correspondence between threads and the map's neurons is assumed.\nBecause of weight sharing, threads inside a block can access data in parallel, in particular the\nsame weights and inputs from the previous layer. Each thread starts by initializing its sum with\nthe bias, then loops over all map connections, convolving the appropriate patch of the input map\nwith the corresponding kernel. The output is obtained by passing the sum through a scaled tanh"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f16a-6db4-11ee-b475-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-10", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "the bias, then loops over all map connections, convolving the appropriate patch of the input map\nwith the corresponding kernel. The output is obtained by passing the sum through a scaled tanh\nactivation function, and then written to device memory.\n3.3 Backward propagation\nBP of deltas can be done in two ways: by pushing or by pulling. Pushing deltas means taking each\ndelta from the current layer and computing the corresponding deltas for the previous layer. For\nan architecture with shared weights this has the disadvantage of being hard to code. Each delta\nfrom the current layer contributes to many deltas in the previous layer, which translates into a lot\nof programming. There are two ways of avoiding this: either writing partial deltas to a separated\nblock of memory and then putting everything together by calling another kernel (slow because\nof a tremendous increase in the number of memory accesses, and the need of another kernel), or\nusing atomic writes (to avoid data hazards) to update deltas (very slow because many writings\nare serialized). We implement pulling deltas, which has almost none of the above speed-limiting\ndrawbacks, but is a bit more complicated.\nTechnical Report No. IDSIA-01-11 5\nThe (uni- or bi-dimensional) thread grid assigns a (bi- or uni-dimensional) thread block to each"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f16b-6db4-11ee-afef-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-11", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Technical Report No. IDSIA-01-11 5\nThe (uni- or bi-dimensional) thread grid assigns a (bi- or uni-dimensional) thread block to each\nmap in the previous layer and a thread to each neuron in every map. Similar to FP, for maps\nwith more than 1024 neurons, the 2D grid is further split into smaller 1D blocks by assigning a\n2D block to each row of the map. Each thread computes the delta of its corresponding neuron by\npulling deltas from the current layer. For every neuron in the previous layer we have to determine\nthe list of neurons in the current layer which are connected to it. Let us consider neuron ( i;j)\nfrom a map in layer Ln\u00001, and then assume that ( x;y) are the coordinates of neurons in maps\nofLnthat contribute to the delta of neuron ( i;j). The (x;y) neuron is connected to kernel size\nnumber neurons ( Kx\u0002Ky) from each connected map in the previous layer. The indices in Ln\u00001\nof the neurons connected through a kernel to the ( x;y) neuron are:\nx(Sx+ 1) \u0014i\u0014x(Sx+ 1) +Kx\u00001;\ny(Sy+ 1) \u0014j\u0014y(Sy+ 1) +Ky\u00001:"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f16c-6db4-11ee-94a8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-12", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "y(Sy+ 1) \u0014j\u0014y(Sy+ 1) +Ky\u00001:\nWe can now compute the inequalities for ( x;y):\ni\u0000Kx+ 1\nSx+ 1\u0014x\u0014i\nSx+ 1;\nj\u0000Ky+ 1\nSy+ 1\u0014y\u0014j\nSy+ 1:\nBecause (x;y) has to be inside the map, the \fnal inequalities are:\nmax\u0012\u0018i\u0000Kx+ 1\nSx+ 1\u0019\n;0\u0013\n\u0014x\u0014min\u0012\u0016i\nSx+ 1\u0017\n;Mx\u00001\u0013\n;\nmax\u0012\u0018j\u0000Ky+ 1\nSy+ 1\u0019\n;0\u0013\n\u0014y\u0014min\u0012\u0016j\nSy+ 1\u0017\n;My\u00001\u0013\n:\nThe above inequalities state that the delta of neuron ( i;j) fromLn\u00001is computed from deltas of\nneurons in a rectangular area in maps of Ln(Fig. 2). After summing up the deltas, each thread\nmultiplies the result by the derivative of the activation function.\n3.4 Adjusting weights\nFP and BP have a grid on the list of maps, but the AW thread grid is on the list of kernels (\flters)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f16d-6db4-11ee-9d05-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-13", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "3.4 Adjusting weights\nFP and BP have a grid on the list of maps, but the AW thread grid is on the list of kernels (\flters)\nbetween maps of two consecutive layers. The 1D grid has a block for each connection between two\nmaps. Thread blocks are 2D, with a corresponding thread for every kernel weight. The bias weight\nis included as an entire row of threads, thus requiring thread blocks to have ( Kx+1)\u0002Kythreads.\nMost of the time these additional Kythreads will do nothing, thread (0,0) being activated only\nfor blocks that have to process the bias.\n4 Experiments\nWe use a system with a Core i7-920 (2.66GHz), 12 GB DDR3 and four graphics cards: 2 x\nGTX 480 and 2 x GTX 580. The correctness of the CPU version is checked by comparing the\nanalytical gradient with its \fnite di\u000berence approximation. On GPU this is not possible because\nall computations are performed with single precision \roating point numbers. Hence the GPU\nimplementation's correctness is checked by comparing its results to those of a randomly initialized\nnet after training it for several epochs on the more accurate CPU version. Obtaining identical\nresults after trillions of operations is a strong indication of correctness.\nTechnical Report No. IDSIA-01-11 6"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f16e-6db4-11ee-9c26-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-14", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "results after trillions of operations is a strong indication of correctness.\nTechnical Report No. IDSIA-01-11 6\nFigure 2: Back propagating deltas. A connection between two maps from two consecutive layers\nis displayed. The map in Ln\u00001has 29 x 29 neurons; the map in Lnhas 13 x 13 neurons. They\nare linked through a 5 x 5 kernel K. Skipping factors of Sx= 1 andSy= 1 are assumed. Arrows\nand colors depict the correspondence between neurons in Ln\u00001and their sources in Ln.\nThe implemented CNN's plain feed-forward architecture is trained using on-line gradient descent. All images from the training set are used for training and also for validation. If deformations\nare enabled, only the images from the training set will be deformed. Weights are initialized according to a uniform random distribution in the range [ \u00000:05;0:05]. Each neuron's activation\nfunction is a scaled hyperbolic tangent: y(a) = 1:7159 tanh(0 :6666a) (LeCun et al. , 1998).\nWe pick the trained CNN with the lowest validation error, and evaluate it on the test set\n(Test for best Validation - TfbV). The best test error (bT) is also listed for all experiments. The"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f16f-6db4-11ee-8b8d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-15", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "(Test for best Validation - TfbV). The best test error (bT) is also listed for all experiments. The\nreported computation times per epoch include training, validation and testing as well as all data\ntransfers.\n4.1 Experiments on MNIST\nFor the MNIST dataset the networks are trained on deformed images, continually generated in\non-line fashion. A\u000ene (translation, rotation, scaling, horizontal shearing) and elastic deformations\n(Simard et al. , 2003) are combined. We use a variable learning rate that shrinks by a multiplicative\nconstant after each epoch, from 10\u00003down to 3 \u000110\u00005after 500 epochs.\nFully connected convolutional layers lead to an exploding number of network connections and\nweights, making training of big and deep CNNs for hundreds of epochs impractical even on GPUs.\nPartial connectivity alleviates this problem and is also biologically more plausible. We reduce\nthe number of connections between convolutional layers in a random way. Table 1 lists results\nof various networks with 2 to 7 hidden layers with random connections. Additional layers result\nin better networks, the best one achieving a test error of 0.35% for best validation and a best\ntest error of 0.27%. The best previous CNN result on MNIST is 0.40% (Simard et al. , 2003)."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f170-6db4-11ee-a943-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-16", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "test error of 0.27%. The best previous CNN result on MNIST is 0.40% (Simard et al. , 2003).\nA 0.35% error rate was recently also obtained by a big, deep MLP (Cire\u0018 san et al. , 2010) with\nmany more free parameters. Deeper nets require more computation time to complete an epoch,\nbut we observe that they also need fewer epochs to achieve good test errors. The deepest CNN\nTechnical Report No. IDSIA-01-11 7\nTable 1: Error rates on MNIST test set for randomly connected CNNs with 2 to 6 convolutional\nlayers with M Maps and an optional fully connected layer with N neurons. Various kernel sizes\nand skipping factors were used.\n#M, #N bT TfbV\nin Hidden Layers [%] [%]\n20M-60M 0.95 1.02\n20M-60M-150N 0.50 0.55\n20M-60M-100M-150N 0.33 0.38\n20M-40M-60M-80M-100M-120M-150N 0.27 0.35\nfrom Table 1 reaches 2.42%, 0.97% and 0.48% after one, three and seventeen epochs, respectively."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f171-6db4-11ee-9e36-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-17", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "from Table 1 reaches 2.42%, 0.97% and 0.48% after one, three and seventeen epochs, respectively.\nOn the other hand, the network with 4 instead of 7 hidden layers reaches 4.71%, 1.58%, 0.68%\nafter one, three and seventeen epochs, achieving a test error below 0.50% after only 34 epochs.\nThis shows once more that deep networks, contrary to common belief, can be trained successfully\nby back-propagation. Despite the numerous free parameters, deep networks seem to learn faster\n(better recognition rates after fewer epochs) than shallow ones.\nWe consider MNIST an almost solved problem. The remaining errors stem from digits that\nare ambiguous or miss parts.\n4.2 Experiments on NORB\nNORB contains stereo images of 3D objects. Hence there are two maps on the input layer.\nRotation, scaling, shearing and elastic distortions seem to have a negative impact on generalization.\nThese deformations improve recognition rates for digits that are intrinsically 2D (Cire\u0018 san et al. ,\n2010), but seem inadequate for 3D objects.\nInitial experiments on NORB show that unlike with MNIST where we use deformations, the\nCNN needs only 3 to 6 epochs to reach zero validation error. This allows us to quickly run numerous"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f172-6db4-11ee-8bf1-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-18", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Initial experiments on NORB show that unlike with MNIST where we use deformations, the\nCNN needs only 3 to 6 epochs to reach zero validation error. This allows us to quickly run numerous\nrepetitive experiments with huge networks with hundreds of maps per layer. We decided to use a\nCNN with \fve hidden layers: layer1, a convolutional layer with 300 maps, kernel size 6 \u00026 and\nskipping factors 1 \u00021; layer2, a max-pooling layer over a 2 \u00022 region; layer3, a convolutional layer\nwith 500 maps, kernel size 4 \u00024, skipping factors 0 \u00020; layer4, a max-pooling layer over a 4 \u00024\nregion; layer5, a fully connected layer with 500 neurons. The learning rate is initialized by 0.001\nand multiplied by 0.95 after every epoch.\nTable 2 summarizes the results of four di\u000berent experiments by switching on/o\u000b translation as\nwell as the \fxed image processing layer. We report the average error rate as well as the standard\ndeviation of N independent runs with identical architectures but di\u000berent weight initializations.\nFor the \frst experiment without translation and no image processing (IP), an average test error\nrate of 7.86% is obtained. With additional translations of at most 5%, the average error rate drops"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f173-6db4-11ee-ae8a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-19", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "rate of 7.86% is obtained. With additional translations of at most 5%, the average error rate drops\nto 4.71%, contradicting the common belief that CNNs are translation invariant. These results are\non par or better than others in the literature: 5.90% error rate for a combination of CNNs and\nSVMs (LeCun et al. , 2004) and 5.20% error rate for restricted Boltzman machines (Nair and\nHinton, 2009).\nThe best previously published result on NORB (2.87%) was obtained by a hierarchical neural\nnetwork which to every convolutional layer provides a subsampled version plus edge information\nof the original image (Uetz and Behnke, 2009). This motivated us to implement a pre-processing\nlayer with \fxed \flters. We tried simple edge masks (Sobel, Scharr) but obtained best results with a\ncontrast-extraction layer (Fukushima, 2003) realized by Mexican hat-shaped \flters of size 21 \u000221,\none with a concentric on-center receptive \feld and one with a concentric o\u000b-center receptive \feld,\nTechnical Report No. IDSIA-01-11 8\nTable 2: Average error rates and standard deviations of N runs for a \fve hidden layer CNN on"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f174-6db4-11ee-8988-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-20", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Technical Report No. IDSIA-01-11 8\nTable 2: Average error rates and standard deviations of N runs for a \fve hidden layer CNN on\nthe NORB test set (see text for details).\ntrans. [%] IP TfbV [%] runs time/epoch [s]\n0 no 7.86 \u00060.55 50 1141\n5 no 4.71 \u00060.57 50 1563\n0 yes 3.94 \u00060.48 50 1658\n5 yes 2.53\u00060.40 100 2080\nsimilar to the \flters automatically created by unsupervised Predictability Minimization (Schmidhuber, 1992) applied to natural images (Schmidhuber et al. , 1996). The \frst \flter extracts positive\ncontrast in brightness, whereas the latter extracts negative contrast. Each image from the original\nNORB is \fltered, consequently the input of the CNN has six maps: the original image plus the\npositive and negative contrast for each of the two stereo channels. Using such a pre-processing\nlayer results in lower average error rates, 3.94% without translation and 2.53% with translation.\nThis result improves the previous state of the art on NORB (Uetz and Behnke, 2009).\nExperience with other image datasets tells us that NORB is unusual. The training set has only"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f175-6db4-11ee-b8f3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-21", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Experience with other image datasets tells us that NORB is unusual. The training set has only\n\fve instances per class. The resulting poor training set variability makes the nets learn quickly\nbut generalize badly. NORB is the only dataset that pro\fts from a \fxed pre-processing layer in\na substantial way. For MNIST and CIFAR10 such pre-processing has little or no e\u000bect. It is also\nworth noting that NORB's standard error rate deviation is bigger than CIFAR10's (see Tables 2\nand 3). Identical nets with di\u000berent initializations do not produce very consistent results. The\nbest net had an error rate of 1.72%, the worst 3.69%.\n4.3 Experiments on CIFAR 10\nCIFAR10 is a collection of natural color images of 32x32 pixels. It contains 10 classes, each of them\nwith 5000 samples in the training set and 1000 in the test set. The images greatly vary inside each\nclass. They are not necessarily centered, may contain only parts of the object, and have varying\nbackgrounds. All of this makes CIFAR10 the hardest problem addressed in this paper. The CNN\nhas three maps, one for each color channel (RGB). The CIFAR10 images are relatively small in"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f176-6db4-11ee-a513-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-22", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "has three maps, one for each color channel (RGB). The CIFAR10 images are relatively small in\ncomparison to NORB's, and force us to use small kernels. The tested CNNs di\u000ber only in the\nnumber of maps per convolutional and max-pooling layer. All have eight hidden layers: layer1, a\nconvolutional layer with 3 \u00023 kernels and skipping factor of 0; layer2, a max-pooling layer over a\n3\u00023 region; layer3, a convolutional layer with 3 \u00023 kernels and skipping factors of 0 \u00020; layer4,\na max-pooling over a 2 \u00022 region; layer5, a convolutional layer with 3 \u00023 kernels and a skipping\nfactors of 0 \u00020; layer6, a max pooling layer over a 2 \u00022 region; layer7, a fully connected layer\nwith 300 neurons; layer8, a fully connected layer with 100 neurons.\nLike for MNIST, the learning rate is initialized by 0.001 and multiplied by 0.993 after every\nepoch. Results in Table 3 show that without translation the error rate does not drop below 28%;\nadding edge information does not help at all. Translations have a very positive e\u000bect, decreasing"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f177-6db4-11ee-8f63-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-23", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "adding edge information does not help at all. Translations have a very positive e\u000bect, decreasing\nthe error rate to almost 20%. Contrast extraction \flters are better than the Sobel/Scharr \flters but\nstill worse than no pre-processing layer at all. Despite some CNN-inherent translation invariance,\nadditional training image translations cause better generalization; additional image processing\nproved useless though.\nTo see if bigger nets are better, we increase the number of maps per layer from 100 to 200, 300\nand 400, respectively (last three rows in Tab. 3). Training time increases exponentially, but the\ntest error decreases, reaching a minimum for nets with 300 maps per layer. Our 19.51% error rate\nis better than the previous state of the art for this dataset, 20.40% (Coates et al. , 2010) and 25.50%\nTechnical Report No. IDSIA-01-11 9\nTable 3: Average error rates and standard deviations for N runs of an eight hidden layer CNN on\nthe CIFAR10 test set (see text for details). The \frst \fve nets have 100 maps per convolutional\nand max-pooling layer, whereas the sixth, seventh and eighth have 200, 300 and 400 maps per"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f178-6db4-11ee-ad57-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-24", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "and max-pooling layer, whereas the sixth, seventh and eighth have 200, 300 and 400 maps per\nhidden layer, respectively. IP - image processing layer: edge - 3 \u00023 Sobel and Scharr \flters; hat 13\u000213 positive and negative contrast extraction \flters.\ntrans. [%] maps IP TfbV [%] runs time/epoch [s]\n0 100 no 28.87 \u00060.37 11 93\n0 100 edge 29.11 \u00060.36 15 104\n5 100 no 20.26 \u00060.21 11 111\n5 100 edge 21.87 \u00060.57 5 120\n5 100 hat 21.44 \u00060.44 4 136\n5 200 no 19.90 \u00060.16 5 248\n5 300 no 19.51 \u00060.18 5 532\n5 400 no 19.54 \u00060.16 5 875\n(Yu and Zhang, 2010). Unlike Coates et al. (2010), however, we use the original images without\nany particular input normalization. Note that the error rate standard deviations are smaller than\nthose obtained on NORB, that is, di\u000berent initializations yield consistent results.\n4.4 Speedup factor of GPU code\nThe GPU code scales well with network size. For small nets the speedup is small (but still over 10)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f179-6db4-11ee-888f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-25", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "4.4 Speedup factor of GPU code\nThe GPU code scales well with network size. For small nets the speedup is small (but still over 10)\nsince they \ft better inside the CPU cache, and GPU resources are underutilized. For huge nets\n(ex: Table 2) the GPU implementation is more than 60 times faster than a compiler-optimized\nCPU version. Given the \rexibility of our GPU version, this is a signi\fcant speedup. One epoch\ntakes 35 GPU minutes but more than 35 CPU hours.\n5 Conclusion\nWe presented high-performance GPU-based CNN variants trained by on-line gradient descent, with\nsparse random connectivity, computationally more e\u000ecient and biologically more plausible than\nfully connected CNNs. Principal advantages include state-of-the-art generalization capabilities,\ngreat \rexibility and speed. All structural CNN parameters such as input image size, number of\nhidden layers, number of maps per layer, kernel sizes, skipping factors and connection tables are\nadaptable to any particular application. We applied our networks to benchmark datasets for digit\nrecognition (MNIST), 3D object recognition (NORB), and natural images (CIFAR10). On MNIST\nthe best network achieved a recognition test error rate of 0.35%, on NORB 2.53% and on CIFAR10"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f17a-6db4-11ee-b6d4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-26", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "the best network achieved a recognition test error rate of 0.35%, on NORB 2.53% and on CIFAR10\n19.51%. Our results are raising the bars for all three benchmarks. Currently the particular CNN\ntypes discussed in this paper seem to be the best adaptive image recognizers, provided there is a\nlabeled dataset of su\u000ecient size. No unsupervised pretraining is required. Good results require\nbig and deep but sparsely connected CNNs, computationally prohibitive on CPUs, but feasible on\ncurrent GPUs, where our implementation is 10 to 60 times faster than a compiler-optimized CPU\nversion.\nTechnical Report No. IDSIA-01-11 10\nAcknowledgment\nThis work was partially funded by the Swiss Commission for Technology and Innovation (CTI),\nProject n. 9688.1 IFF: Intelligent Fill in Form.\nReferences\nS. Behnke. Hierarchical Neural Networks for Image Interpretation , volume 2766 of Lecture Notes\nin Computer Science . Springer, 2003.\nK. Chellapilla, S. Puri, and P. Simard. High performance convolutional neural networks for\ndocument processing. In International Workshop on Frontiers in Handwriting Recognition ,\n2006.\nD. C. Cire\u0018 san, U. Meier, L. M. Gambardella, and J. Schmidhuber. Deep big simple neural nets"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f17b-6db4-11ee-9328-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-27", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "2006.\nD. C. Cire\u0018 san, U. Meier, L. M. Gambardella, and J. Schmidhuber. Deep big simple neural nets\nfor handwritten digit recogntion. Neural Computation , 22(12):3207{3220, 2010.\nA. Coates, H. Lee, and A. Ng. An analysis of single-layer networks in unsupervised feature learning.\nInAdvances in Neural Information Processing Systems , 2010.\nK. Fukushima. Neocognitron: A self-organizing neural network for a mechanism of pattern recognition una\u000bected by shift in position. Biological Cybernetics , 36(4):193{202, 1980.\nK. Fukushima. Neocognitron for handwritten digit recognition. Neurocomputing , 51:161{180,\n2003.\nP. O. Hoyer and A. Hyv\u007f arinen. Independent component analysis applied to feature extraction\nfrom colour and stero images. Network: Computation in Neural Systems , 11(3):191{210, 2000.\nA. Krizhevsky. Learning multiple layers of features from tiny images. Master's thesis, Computer\nScience Department, University of Toronto, 2009.\nY. LeCun, L. Bottou, Y. Bengio, and P. Ha\u000bner. Gradient-based learning applied to document"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f17c-6db4-11ee-8823-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-28", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "Y. LeCun, L. Bottou, Y. Bengio, and P. Ha\u000bner. Gradient-based learning applied to document\nrecognition. Proceedings of the IEEE , 86(11):2278{2324, November 1998.\nY. LeCun, F.-J. Huang, and L. Bottou. Learning methods for generic object recognition with invariance to pose and lighting. In Proc. of Computer Vision and Pattern Recognition Conference ,\n2004.\nJ. Mutch and D. G. Lowe. Object class recognition and localization using sparse features with\nlimited receptive \felds. Int. J. Comput. Vision , 56(6):503{511, 2008.\nV. Nair and G. E. Hinton. 3d object recognition with deep belief nets. In Advances in Neural\nInformation Processing Systems , 2009.\nB. A. Olshausen and D. J. Field. Sparse coding with an overcomplete basis set: A strategy\nemployed by v1? Vision Research , 37(23):3311{3325, December 1997.\nN. Pinto, D. Doukhan, J. J. DiCarlo, and D. D. Cox. A high-throughput screening approach\nto discovering good forms of biologically inspired visual representation. PLoS computational\nbiology , 5(11):e1000579, November 2009."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f17d-6db4-11ee-85b0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-29", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "to discovering good forms of biologically inspired visual representation. PLoS computational\nbiology , 5(11):e1000579, November 2009.\nM. Riesenhuber and T. Poggio. Hierarchical models of object recognition in cortex. Nat. Neurosci. ,\n2(11):1019{1025, 1999.\nTechnical Report No. IDSIA-01-11 11\nD. Scherer, A. M\u007f uller, and S. Behnke. Evaluation of pooling operations in convolutional architectures for object recognition. In International Conference on Arti\fcial Neural Networks , 2010.\nJ. Schmidhuber, M. Eldracher, and B. Foltin. Semilinear predictability minimization produces\nwell-known feature detectors. Neural Computation , 8(4):773{786, 1996.\nJ. Schmidhuber. Learning factorial codes by predictability minimization. Neural Computation ,\n4(6):863{879, 1992.\nT. Serre, L. Wolf, and T. Poggio. Object recognition with features inspired by visual cortex. In\nProc. of Computer Vision and Pattern Recognition Conference , 2007.\nP. Simard, D. Steinkraus, and J. Platt. Best practices for convolutional neural networks applied\nto visual document analysis. In Seventh International Conference on Document Analysis and"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f17e-6db4-11ee-b130-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1102.0183-30", "title": "High-Performance Neural Networks for Visual Object Classification", "source": "http://arxiv.org/pdf/1102.0183", "start_index": 0, "chroma:document": "to visual document analysis. In Seventh International Conference on Document Analysis and\nRecognition , pages 958{963, 2003.\nR. Uetz and S. Behnke. Large-scale object recognition with cuda-accelerated hierarchical neural\nnetworks. In IEEE International Converence on Intelligent Computing and Intelligent Systems\n(ICIS) , 2009.\nD. H. Wiesel and T. N. Hubel. Receptive \felds of single neurones in the cat's striate cortex. J.\nPhysiol. , 148:574{591, 1959.\nK. Yu and T. Zhang. Improved local coordinate coding using local tangents. In Proceedings of the\nInternational Conference on Machine Learning , 2010."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f17f-6db4-11ee-a16f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-0", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Knowledge Matters: Importance of Prior Information for\nOptimization\nC \u0018 a\u0015 glar G\u007f ul\u0018 cehre gulcehrc@iro.umontreal.ca\nD\u0013 epartement d'informatique et de recherche op\u0013 erationnelle\nUniversit\u0013 e de Montr\u0013 eal, Montr\u0013 eal, QC, Canada\nYoshua Bengio bengioy@iro.umontreal.ca\nD\u0013 epartement d'informatique et de recherche op\u0013 erationnelle\nUniversit\u0013 e de Montr\u0013 eal, Montr\u0013 eal, QC, Canada\nEditor: Not Assigned\nAbstract\nWe explore the e\u000bect of introducing prior information into the intermediate level of deep\nsupervised neural networks for a learning task on which all the black-box state-of-the-art machine learning algorithms tested have failed to learn. We motivate our work from the hypothesis\nthat there is an optimization obstacle involved in the nature of such tasks, and that humans\nlearn useful intermediate concepts from other individuals via a form of supervision or guidance\nusing a curriculum. The experiments we have conducted provide positive evidence in favor of\nthis hypothesis. In our experiments, a two-tiered MLP architecture is trained on a dataset for"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f180-6db4-11ee-84b5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-1", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "using a curriculum. The experiments we have conducted provide positive evidence in favor of\nthis hypothesis. In our experiments, a two-tiered MLP architecture is trained on a dataset for\nwhich each image input contains three sprites, and the binary target class is 1 if all three have\nthe same shape. Black-box machine learning algorithms only got chance on this task. Standard\ndeep supervised neural networks also failed. However, using a particular structure and guiding\nthe learner by providing intermediate targets in the form of intermediate concepts (the presence of each object) allows to nail the task. Much better than chance but imperfect results are\nalso obtained by exploring architecture and optimization variants, pointing towards a di\u000ecult\noptimization task. We hypothesize that the learning di\u000eculty is due to the composition of two\nhighly non-linear tasks. Our \fndings are also consistent with hypotheses on cultural learning\ninspired by the observations of e\u000bective local minima (possibly due to ill-conditioning and the\ntraining procedure not being able to escape what appears like a local minimum).\nKeywords: Deep Learning, Neural Networks, Optimization, Evolution of Culture, Curriculum Learning, Training with Hints\n1. Introduction\nThere is a recent emerging interest in di\u000berent \felds of science for cultural learning (Henrich\nand McElreath, 2003) and how groups of individuals exchanging information can learn in ways"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f181-6db4-11ee-9f0e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-2", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "and McElreath, 2003) and how groups of individuals exchanging information can learn in ways\nsuperior to individual learning. This is also witnessed by the emergence of new research \felds\nsuch as \"Social Neuroscience\". Learning from other agents in an environment by the means\nof cultural transmission of knowledge with a peer-to-peer communication is an e\u000ecient and\nnatural way of acquiring or propagating common knowledge. The most popular belief on how\nthe information is transmitted between individuals is that bits of information are transmitted\nby small units, called memes, which share some characteristics of genes, such as self-replication,\nmutation and response to selective pressures (Dawkins, 1976).\n1arXiv:1301.4083v6  [cs.LG]  13 Jul 2013\nThis paper is based on the hypothesis (which is further elaborated in Bengio (2013a)) that\nhuman culture and the evolution of ideas have been crucial to counter an optimization issue:\nthis di\u000eculty would otherwise make it di\u000ecult for human brains to capture high level knowledge of the world without the help of other educated humans. In this paper machine learning\nexperiments are used to investigate some elements of this hypothesis by seeking answers for\nthe following questions: are there machine learning tasks which are intrinsically hard for a\nlone learning agent but that may become very easy when intermediate concepts are provided"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f182-6db4-11ee-9a3c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-3", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "the following questions: are there machine learning tasks which are intrinsically hard for a\nlone learning agent but that may become very easy when intermediate concepts are provided\nby another agent as additional intermediate learning cues, in the spirit of Curriculum Learning (Bengio et al., 2009b)? What makes such learning tasks more di\u000ecult? Can speci\fc initial\nvalues of the neural network parameters yield success when random initialization yield complete failure? Is it possible to verify that the problem being faced is an optimization problem or\nwith a regularization problem? These are the questions discussed (if not completely addressed)\nhere, which relate to the following broader question: how can humans (and potentially one day,\nmachines) learn complex concepts?\nIn this paper, results of di\u000berent machine learning algorithms on an arti\fcial learning task\ninvolving binary 64 \u000264 images are presented. In that task, each image in the dataset contains\n3 Pentomino tetris sprites (simple shapes). The task is to \fgure out if all the sprites in the\nimage are the same or if there are di\u000berent sprite shapes in the image. Several state-of-the-art\nmachine learning algorithms have been tested and none of them could perform better than\na random predictor on the test set. Nevertheless by providing hints about the intermediate\nconcepts (the presence and location of particular sprite classes), the problem can easily be solved"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f183-6db4-11ee-a1f7-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-4", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "a random predictor on the test set. Nevertheless by providing hints about the intermediate\nconcepts (the presence and location of particular sprite classes), the problem can easily be solved\nwhere the same-architecture neural network without the intermediate concepts guidance fails.\nSurprisingly, our attempts at solving this problem with unsupervised pre-training algorithms\nfailed solve this problem. However, with speci\fc variations in the network architecture or\ntraining procedure, it is found that one can make a big dent in the problem. For showing the\nimpact of intermediate level guidance, we experimented with a two-tiered neural network, with\nsupervised pre-training of the \frst part to recognize the category of sprites independently of\ntheir orientation and scale, at di\u000berent locations, while the second part learns from the output\nof the \frst part and predicts the binary task of interest.\nThe objective of this paper is not to propose a novel learning algorithm or architecture,\nbut rather to re\fne our understanding of the learning di\u000eculties involved with composed tasks\n(here a logical formula composed with the detection of object classes), in particular the training\ndi\u000eculties involved for deep neural networks. The results also bring empirical evidence in favor\nof some of the hypotheses from Bengio (2013a), discussed below, as well as introducing a\nparticular form of curriculum learning (Bengio et al., 2009b)."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f184-6db4-11ee-8d9a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-5", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "of some of the hypotheses from Bengio (2013a), discussed below, as well as introducing a\nparticular form of curriculum learning (Bengio et al., 2009b).\nBuilding di\u000ecult AI problems has a long history in computer science. Speci\fcally hard\nAI problems have been studied to create CAPTCHA's that are easy to solve for humans, but\nhard to solve for machines (Von Ahn et al., 2003). In this paper we are investigating a di\u000ecult\nproblem for the o\u000b-the-shelf black-box machine learning algorithms.1\n1.1 Curriculum Learning and Cultural Evolution Against E\u000bective Local Minima\nWhat Bengio (2013a) calls an e\u000bective local minimum is a point where iterative training\nstalls, either because of an actual local minimum or because the optimization algorithm is\n1. You can access the source code of some experiments presented in that paper and their hyperparameters from\nhere: https://github.com/caglar/kmatters\n2\nunable (in reasonable time) to \fnd a descent path (e.g., because of serious ill-conditioning). In\nthis paper, it is hypothesized that some more abstract learning tasks such as those obtained\nby composing simpler tasks are more likely to yield e\u000bective local minima for neural networks,\nand are generally hard for general-purpose machine learning algorithms."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f185-6db4-11ee-bf78-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-6", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "by composing simpler tasks are more likely to yield e\u000bective local minima for neural networks,\nand are generally hard for general-purpose machine learning algorithms.\nThe idea that learning can be enhanced by guiding the learner through intermediate easier\ntasks is old, starting with animal training by shaping (Skinner, 1958; Peterson, 2004; Krueger\nand Dayan, 2009). Bengio et al. (2009b) introduce a computational hypothesis related to a\npresumed issue with e\u000bective local minima when directly learning the target task: the good\nsolutions correspond to hard-to-\fnd-by-chance e\u000bective local minima, and intermediate tasks\nprepare the learner's internal con\fguration (parameters) in a way similar to continuation methods in global optimization (which go through a sequence of intermediate optimization problems,\nstarting with a convex one where local minima are no issue, and gradually morphing into the\ntarget task of interest).\nIn a related vein, Bengio (2013a) makes the following inferences based on experimental\nobservations of deep learning and neural network learning:\nPoint 1: Training deep architectures is easier when some hints are given about the function\nthat the intermediate levels should compute (Hinton et al., 2006; Weston et al., 2008;\nSalakhutdinov and Hinton, 2009; Bengio, 2009). The experiments performed here expand"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f186-6db4-11ee-87e0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-7", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Salakhutdinov and Hinton, 2009; Bengio, 2009). The experiments performed here expand\nin particular on this point.\nPoint 2: It is much easier to train a neural network with supervision (where examples ar\nprovided to it of when a concept is present and when it is not present in a variety of\nexamples) than to expect unsupervised learning to discover the concept (which may also\nhappen but usually leads to poorer renditions of the concept). The poor results obtained\nwith unsupervised pre-training reinforce that hypothesis .\nPoint 3: Directly training all the layers of a deep network together not only makes it di\u000ecult to\nexploit all the extra modeling power of a deeper architecture but in many cases it actually\nyields worse results as the number of required layers is increased (Larochelle et al., 2009;\nErhan et al., 2010). The experiments performed here also reinforce that hypothesis.\nPoint 4: Erhan et al. (2010) observed that no two training trajectories ended up in the\nsame e\u000bective local minimum, out of hundreds of runs, even when comparing solutions as\nfunctions from input to output, rather than in parameter space (thus eliminating from the\npicture the presence of symmetries and multiple local minima due to relabeling and other"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f187-6db4-11ee-ab1d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-8", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "functions from input to output, rather than in parameter space (thus eliminating from the\npicture the presence of symmetries and multiple local minima due to relabeling and other\nreparametrizations). This suggests that the number of di\u000berent e\u000bective local minima\n(even when considering them only in function space) must be huge.\nPoint 5: Unsupervised pre-training, which changes the initial conditions of the descent procedure, sometimes allows to reach substantially better e\u000bective local minima (in terms\nof generalization error!), and these better local minima do not appear to be reachable\nby chance alone (Erhan et al., 2010). The experiments performed here provide another\npiece of evidence in favor of the hypothesis that where random initialization can yield\nrather poor results, speci\fcally targeted initialization can have a drastic impact, i.e., that\n3\ne\u000bective local minima are not just numerous but that some small subset of them are much\nbetter and hard to reach by chance.2\nBased on the above points, Bengio (2013a) then proposed the following hypotheses regarding\nlearning of high-level abstractions.\n\u000fOptimization Hypothesis: When it learns, a biological agent performs an approximate\noptimization with respect to some implicit objective function."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f188-6db4-11ee-a03a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-9", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "learning of high-level abstractions.\n\u000fOptimization Hypothesis: When it learns, a biological agent performs an approximate\noptimization with respect to some implicit objective function.\n\u000fDeep Abstractions Hypothesis: Higher level abstractions represented in brains require deeper computations (involving the composition of more non-linearities).\n\u000fLocal Descent Hypothesis: The brain of a biological agent relies on approximate local\ndescent and gradually improves itself while learning.\n\u000fE\u000bective Local Minima Hypothesis: The learning process of a single human learner\n(not helped by others) is limited by e\u000bective local minima.\n\u000fDeeper Harder Hypothesis: E\u000bective local minima are more likely to hamper learning\nas the required depth of the architecture increases.\n\u000fAbstractions Harder Hypothesis: High-level abstractions are unlikely to be discovered by a single human learner by chance, because these abstractions are represented by\na deep subnetwork of the brain, which learns by local descent.\n\u000fGuided Learning Hypothesis: A human brain can learn high level abstractions if\nguided by the signals produced by other agents that act as hints or indirect supervision\nfor these high-level abstractions.\n\u000fMemes Divide-and-Conquer Hypothesis: Linguistic exchange, individual learning"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f189-6db4-11ee-a2e2-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-10", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "for these high-level abstractions.\n\u000fMemes Divide-and-Conquer Hypothesis: Linguistic exchange, individual learning\nand the recombination of memes constitute an e\u000ecient evolutionary recombination operator in the meme-space. This helps human learners to collectively build better internal\nrepresentations of their environment, including fairly high-level abstractions.\nThis paper is focused on \\ Point 1 \" and testing the \\ Guided Learning Hypothesis \", using\nmachine learning algorithms to provide experimental evidence. The experiments performed\nalso provide evidence in favor of the \\ Deeper Harder Hypothesis \" and associated \\ Abstractions\nHarder Hypothesis \". Machine Learning is still far beyond the current capabilities of humans,\nand it is important to tackle the remaining obstacles to approach AI. For this purpose, the\nquestion to be answered is why tasks that humans learn e\u000bortlessly from very few examples,\nwhile machine learning algorithms fail miserably?\n2. Recent work showed that rather deep feedforward networks can be very successfully trained when large\nquantities of labeled data are available (Ciresan et al., 2010; Glorot et al., 2011a; Krizhevsky et al., 2012).\nNonetheless, the experiments reported here suggest that it all depends on the task being considered, since\neven with very large quantities of labeled examples, the deep networks trained here were unsuccessful.\n4"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f18a-6db4-11ee-a1ea-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-11", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Nonetheless, the experiments reported here suggest that it all depends on the task being considered, since\neven with very large quantities of labeled examples, the deep networks trained here were unsuccessful.\n4\n2. Culture and Optimization Di\u000eculty\nAs hypothesized in the \\ Local Descent Hypothesis \", human brains would rely on a local approximate descent, just like a Multi-Layer Perceptron trained by a gradient-based iterative optimization. The main argument in favor of this hypothesis relies on the biologically-grounded\nassumption that although \fring patterns in the brain change rapidly, synaptic strengths underlying these neural activities change only gradually, making sure that behaviors are generally\nconsistent across time. If a learning algorithm is based on a form of local (e.g. gradient-based)\ndescent, it can be sensitive to e\u000bective local minima (Bengio, 2013a).\nWhen one trains a neural network, at some point in the training phase the evaluation of\nerror seems to saturate, even if new examples are introduced. In particular Erhan et al. (2010)\n\fnd that early examples have a much larger weight in the \fnal solution. It looks like the learner\nis stuck in or near a local minimum. But since it is di\u000ecult to verify if this is near a true local"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f18b-6db4-11ee-a427-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-12", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "is stuck in or near a local minimum. But since it is di\u000ecult to verify if this is near a true local\nminimum or simply an e\u000bect of strong ill-conditioning, we call such a \\stuck\" con\fguration an\ne\u000bective local minimum , whose de\fnition depends not just on the optimization objective but\nalso on the limitations of the optimization algorithm.\nErhan et al. (2010) highlighted both the issue of e\u000bective local minima and a regularization e\u000bect when initializing a deep network with unsupervised pre-training. Interestingly, as\nthe network gets deeper the di\u000eculty due to e\u000bective local minima seems to be get more pronounced. That might be because of the number of e\u000bective local minima increases (more like\nan actual local minima issue), or maybe because the good ones are harder to reach (more like\nan ill-conditioning issue) and more work will be needed to clarify this question.\nAs a result of Point 4 we hypothesize that it is very di\u000ecult for an individual's brain to\ndiscover some higher level abstractions by chance only. As mentioned in the \\ Guided Learning\nHypothesis \" humans get hints from other humans and learn high-level concepts with the guidance of other humans3. Curriculum learning (Bengio et al., 2009a) and incremental learning"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f18c-6db4-11ee-b7b5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-13", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "(Solomono\u000b, 1989), are examples of this. This is done by properly choosing the sequence of\nexamples seen by the learner, where simpler examples are introduced \frst and more complex\nexamples shown when the learner is ready for them. One of the hypothesis on why curriculum\nworks states that curriculum learning acts as a continuation method that allows one to discover\na good minimum, by \frst \fnding a good minimum of a smoother error function. Recent experiments on human subjects also indicates that humans teach by using a curriculum strategy\n(Khan et al., 2011).\nSome parts of the human brain are known to have a hierarchical organization (i.e. visual\ncortex) consistent with the deep architecture studied in machine learning papers. As we go from\nthe sensory level to higher levels of the visual cortex, we \fnd higher level areas corresponding\nto more abstract concepts. This is consistent with the Deep Abstractions Hypothesis .\nTraining neural networks and machine learning algorithms by decomposing the learning\ntask into sub-tasks and exploiting prior information about the task is well-established and\nin fact constitutes the main approach to solving industrial problems with machine learning.\nThe contribution of this paper is rather on rendering explicit the e\u000bective local minima issue\nand providing evidence on the type of problems for which this di\u000eculty arises. This prior"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f18d-6db4-11ee-9fdd-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-14", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "The contribution of this paper is rather on rendering explicit the e\u000bective local minima issue\nand providing evidence on the type of problems for which this di\u000eculty arises. This prior\ninformation and hints given to the learner can be viewed as inductive bias for a particular task,\nan important ingredient to obtain a good generalization error (Mitchell, 1980). An interesting\n3. But some high-level concepts may also be hardwired in the brain, as assumed in the universal grammar\nhypothesis (Montague, 1970), or in nature vs nurture discussions in cognitive science.\n5\nearlier \fnding in that line of research was done with Explanation Based Neural Networks\n(EBNN) in which a neural network transfers knowledge across multiple learning tasks. An\nEBNN uses previously learned domain knowledge as an initialization or search bias (i.e. to\nconstrain the learner in the parameter space) (O'Sullivan, 1996; Mitchell and Thrun, 1993).\nAnother related work in machine learning is mainly focused on reinforcement learning algorithms, based on incorporating prior knowledge in terms of logical rules to the learning\nalgorithm as a prior knowledge to speed up and bias learning (Kunapuli et al., 2010; Towell\nand Shavlik, 1994).\nAs discussed in \\ Memes Divide and Conquer Hypothesis \\ societies can be viewed as a"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f18e-6db4-11ee-93f5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-15", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "and Shavlik, 1994).\nAs discussed in \\ Memes Divide and Conquer Hypothesis \\ societies can be viewed as a\ndistributed computational processing systems. In civilized societies knowledge is distributed\nacross di\u000berent individuals, this yields a space e\u000eciency. Moreover computation, i.e. each\nindividual can specialize on a particular task/topic, is also divided across the individuals in the\nsociety and hence this will yield a computational e\u000eciency. Considering the limitations of the\nhuman brain, the whole processing can not be done just by a single agent in an e\u000ecient manner.\nA recent study in paleoantropology states that there is a substantial decline in endocranial\nvolume of the brain in the last 30000 years Henneberg (1988). The volume of the brain shrunk\nto 1241 ml from 1502 ml (Henneberg and Steyn, 1993). One of the hypothesis on the reduction\nof the volume of skull claims that, decline in the volume of the brain might be related to the\nfunctional changes in brain that arose as a result of cultural development and emergence of\nsocieties given that this time period overlaps with the transition from hunter-gatherer lifestyle\nto agricultural societies.\n3. Experimental Setup\nSome tasks, which seem reasonably easy for humans to learn4, are nonetheless appearing almost\nimpossible to learn for current generic state-of-art machine learning algorithms."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f18f-6db4-11ee-ac15-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-16", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "3. Experimental Setup\nSome tasks, which seem reasonably easy for humans to learn4, are nonetheless appearing almost\nimpossible to learn for current generic state-of-art machine learning algorithms.\nHere we study more closely such a task, which becomes learnable if one provides hints to\nthe learner about appropriate intermediate concepts. Interestingly, the task we used in our\nexperiments is not only hard for deep neural networks but also for non-parametric machine\nlearning algorithms such as SVM's, boosting and decision trees.\nThe result of the experiments for varying size of dataset with several o\u000b-the-shelf black box\nmachine learning algorithms and some popular deep learning algorithms are provided in Table\n1. The detailed explanations about the algorithms and the hyperparameters used for those\nalgorithms are given in the Appendix Section 5.2. We also provide some explanations about\nthe methodologies conducted for the experiments at Section 3.2.\n3.1 Pentomino Dataset\nIn order to test our hypothesis, an arti\fcial dataset for object recognition using 64 \u000264 binary\nimages is designed5. If the task is two tiered (i.e., with guidance provided), the task in the\n\frst part is to recognize and locate each Pentomino object class6in the image. The second\n4. keeping in mind that humans can exploit prior knowledge, either from previous learning or innate knowledge."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f190-6db4-11ee-b359-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-17", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "4. keeping in mind that humans can exploit prior knowledge, either from previous learning or innate knowledge.\n5. The source code for the script that generates the arti\fcial Pentomino datasets (Arcade-Universe) is available\nat:https://github.com/caglar/Arcade-Universe . This implementation is based on Olivier Breuleux's\nbugland dataset generator.\n6. A human learner does not seem to need to be taught the shape categories of each Pentomino sprite in order\nto solve the task. On the other hand, humans have lots of previously learned knowledge about the notion of\nshape and how central it is in de\fning categories.\n6\n(a) sprites, not all same type\n (b) sprites, all of same type\nFigure 1: Left (a): An example image from the dataset which has a di\u000berent sprite type in it.\nRight (b): An example image from the dataset that has only one type of Pentomino\nobject in it, but with di\u000berent orientations and scales.\npart/\fnal binary classi\fcation task is to \fgure out if all the Pentominos in the image are of\nthe same shape class or not. If a neural network learned to detect the categories of each object\nat each location in an image, the remaining task becomes an XOR-like operation between the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f191-6db4-11ee-bd4a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-18", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "the same shape class or not. If a neural network learned to detect the categories of each object\nat each location in an image, the remaining task becomes an XOR-like operation between the\ndetected object categories. The types of Pentomino objects that is used for generating the\ndataset are as follows:\nPentomino sprites N, P, F, Y, J, and Q, along with the Pentomino N2 sprite (mirror of\n\\Pentomino N\" sprite), the Pentomino F2 sprite (mirror of \\Pentomino F\" sprite), and the\nPentomino Y2 sprite (mirror of \\Pentomino Y\" sprite).\nFigure 2: Di\u000berent classes of Pentomino shapes used in our dataset.\nAs shown in Figures 1(a) and 1(b), the synthesized images are fairly simple and do not\nhave any texture. Foreground pixels are \\1\" and background pixels are \\0\". Images of the\ntraining and test sets are generated iid. For notational convenience, assume that the domain\nof raw input images is X, the set of sprites is S, the set of intermediate object categories is Y\nfor each possible location in the image and the set of \fnal binary task outcomes is Z=f0;1g."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f192-6db4-11ee-8586-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-19", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "for each possible location in the image and the set of \fnal binary task outcomes is Z=f0;1g.\nTwo di\u000berent types of rigid body transformation is performed: sprite rotation rot(X;\r) where\n\u0000 =f\r: (\r= 90\u0002\u001e)^[(\u001e2N);(0\u0014\u001e\u00143)]gand scaling scale (X;\u000b) where\u000b2f1;2gis the\nscaling factor. The data generating procedure is summarized below.\nSprite transformations: Before placing the sprites in an empty image, for each image x2X,\na value for z2Zis randomly sampled which is to have (or not) the same three sprite\nshapes in the image. Conditioned on the constraint given by z, three sprites are randomly\n7\nselectedsijfromSwithout replacement. Using a uniform probability distribution over\nall possible scales, a scale is chosen and accordingly each sprite image is scaled. Then\nrotate each sprite is randomly rotated by a multiple of 90 degrees.\nSprite placement: Upon completion of sprite transformations, a 64 \u000264 uniform grid is generated which is divided into 8 \u00028 blocks, each block being of size 8 \u00028 pixels, and randomly\nselect three di\u000berent blocks from the 64=8 \u00028 on the grid and place the transformed"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f193-6db4-11ee-ba7c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-20", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "select three di\u000berent blocks from the 64=8 \u00028 on the grid and place the transformed\nobjects into di\u000berent blocks (so they cannot overlap, by construction).\nEach sprite is centered in the block in which it is located. Thus there is no object translation\ninside the blocks. The only translation invariance is due to the location of the block inside the\nimage.\nA Pentomino sprite is guaranteed to not over\row the block in which it is located, and there\nare no collisions or overlaps between sprites, making the task simpler. The largest possible\nPentomino sprite can be \ft into an 8 \u00024 mask.\n3.2 Learning Algorithms Evaluated\nInitially the models are cross-validated by using 5-fold cross-validation. With 40,000 examples,\nthis gives 32,000 examples for training and 8,000 examples for testing. For neural network\nalgorithms, stochastic gradient descent (SGD) is used for training. The following standard\nlearning algorithms were \frst evaluated: decision trees, SVMs with Gaussian kernel, ordinary\nfully-connected Multi-Layer Perceptrons, Random Forests, k-Nearest Neighbors, Convolutional\nNeural Networks, and Stacked Denoising Auto-Encoders with supervised \fne-tuning. More"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f194-6db4-11ee-ade1-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-21", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Neural Networks, and Stacked Denoising Auto-Encoders with supervised \fne-tuning. More\ndetails of the con\fgurations and hyper-parameters for each of them are given in Appendix\nSection 5.2. The only better than chance results were obtained with variations of the Structured\nMulti-Layer Perceptron described below.\n3.2.1 Structured Multi-Layer Perceptron (SMLP)\nThe neural network architecture that is used to solve this task is called the SMLP (Structured\nMulti-Layer Perceptron), a deep neural network with two parts as illustrated in Figure 5 and\n7:\nThe lower part, P1NN ( Part 1 Neural Network , as it is called in the rest of the paper),\nhas shared weights and local connectivity, with one identical MLP instance of the P1NN for\neach patch of the image, and typically an 11-element output vector per patch (unless otherwise\nnoted). The idea is that these 11 outputs per patch could represent the detection of the sprite\nshape category (or the absence of sprite in the patch). The upper part, P2NN ( Part 2 Neural\nNetwork ) is a fully connected one hidden layer MLP that takes the concatenation of the outputs"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f195-6db4-11ee-aa9d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-22", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Network ) is a fully connected one hidden layer MLP that takes the concatenation of the outputs\nof all patch-wise P1NNs as input. Note that the \frst layer of P1NN is similar to a convolutional\nlayer but where the stride equals the kernel size, so that windows do not overlap, i.e., P1NN can\nbe decomposed into separate networks sharing the same parameters but applied on di\u000berent\npatches of the input image, so that each network can actually be trained patch-wise in the case\nwhere a target is provided for the P1NN outputs. The P1NN output for patch piwhich is\nextracted from the image xis computed as follows:\nf\u0012(pi) =g2(Vg1(Upi+b) +c) (1)\n8\nwhere pi2Rdis the input patch/receptive \feld extracted from location iof a single image.\nU2Rdh\u0002dis the weight matrix for the \frst layer of P1NN and b2Rd\nhis the vector of biases\nfor the \frst layer of P1NN. g1(\u0001) is the activation function of the \frst layer and g2(\u0001) is the\nactivation function of the second layer. In many of the experiments, best results were obtained"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f196-6db4-11ee-884b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-23", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "activation function of the second layer. In many of the experiments, best results were obtained\nwithg1(\u0001) a rectifying non-linearity (a.k.a. as RELU), which is max(0; X) (Jarrett et al.,\n2009b; Nair and Hinton, 2010; Glorot et al., 2011a; Krizhevsky et al., 2012). V2Rdh\u0002dois the\nsecond layer's weights matrix, such that and c2Rdoare the biases of the second layer of the\nP1NN, with doexpected to be smaller than dh.\nIn this way, g1(Upi+b) is an overcomplete representation of the input patch that can\npotentially represent all the possible Pentomino shapes for all factors of variations in the patch\n(rotation, scaling and Pentomino shape type). On the other hand, when trained with hints,\nf\u0012(pi) is expected to be the lower dimensional representation of a Pentomino shape category\ninvariant to scaling and rotation in the given patch.\nIn the experiments with SMLP trained with hints (targets at the output of P1NN), the\nP1NN is expected to perform classi\fcation of each 8 \u00028 non-overlapping patches of the original"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f197-6db4-11ee-bdbd-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-24", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "P1NN is expected to perform classi\fcation of each 8 \u00028 non-overlapping patches of the original\n64\u000264 input image without having any prior knowledge of whether that speci\fc patch contains\na Pentomino shape or not. P1NN in SMLP without hints just outputs the local activations for\neach patch, and gradients on f\u0012(pi) are backpropagated from the upper layers. In both cases\nP1NN produces the input representation for the Part 2 Neural Net (P2NN). Thus the input\nrepresentation of P2NN is the concatenated output of P1NN across all the 64 patch locations:\nho= [f\u0012(p0);:::;f\u0012(pi);:::;f\u0012(pN))] whereNis the number of patches and the ho2Rdi;di=\ndo\u0002N.hois the concatenated output of the P1NN at each patch.\nThere is a standardization layer on top of the output of P1NN that centers the activations\nand performs divisive normalization by dividing by the standard deviation over a minibatch\nof the activations of that layer. We denote the standardization function z(\u0001). Standardization\nmakes use of the mean and standard deviation computed for each hidden unit such that each"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f198-6db4-11ee-b4fc-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-25", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "of the activations of that layer. We denote the standardization function z(\u0001). Standardization\nmakes use of the mean and standard deviation computed for each hidden unit such that each\nhidden unit of howill have 0 activation and unit standard deviation on average over the\nminibatch. Xis the set of pentomino images in the minibatch, where X2Rdin\u0002Nis a matrix\nwithNimages.h(i)\no(xj) is the vector of activations of the i-th hidden unit of hidden layer\nho(xj) for thej-th example, with xj2X.\n\u0016h(i)\no=1\nNX\nxj2Xh(i)\no(xj) (2)\n\u001bh(i)\no=sPN\nj(h(i)\no(xj)\u0000\u0016h(i)\no)2\nN+\u000f (3)\nz(h(i)\no(xj)) =h(i)\no(xj)\u0000\u0016h(i)\no\nmax(\u001bh(i)\no;\u000f)(4)\nwhere\u000fis a very small constant, that is used to prevent numerical under\rows in the standard\ndeviation. P1NN is trained on each 8 \u00028 patches extracted from the image."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f199-6db4-11ee-a4c3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-26", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "deviation. P1NN is trained on each 8 \u00028 patches extracted from the image.\nhois standardized for each training and test sample separately. Di\u000berent values of \u000fwere\nused for SMLP-hints and SMLP-nohints.\nThe concatenated output of P1NN is fed as an input to the P2NN. P2NN is a feedforward\nMLP with a sigmoid output layer using a single RELU hidden layer. The task of P2NN is to\nperform a nonlinear logical operation on the representation provided at the output of P1NN.\n9\n3.2.2 Structured Multi Layer Perceptron Trained with Hints (SMLP-hints)\nThe SMLP-hints architecture exploits a hint about the presence and category of Pentomino\nobjects, specifying a semantics for the P1NN outputs. P1NN is trained with the intermediate\ntargetY, specifying the type of Pentomino sprite shape present (if any) at each of the 64 patches\n(8\u00028 non-overlapping blocks) of the image. Because a possible answer at a given location can\nbe \\none of the object types\" i.e., an empty patch, yp(for patchp) can take one of the 11"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f19a-6db4-11ee-8c1c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-27", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "be \\none of the object types\" i.e., an empty patch, yp(for patchp) can take one of the 11\npossible values, 1 for rejection and the rest is for the Pentomino shape classes, illustrated in\nFigure 2:\nyp=(\n0 if patch pis empty\ns2Sif the patch pcontains a Pentomino sprite :\nA similar task has been studied by Fleuret et al. (2011) (at SI appendix Problem 17), who\ncompared the performance of humans vs computers.\nThe SMLP-hints architecture takes advantage of dividing the task into two subtasks during\ntraining with prior information about intermediate-level relevant factors. Because the sum of\nthe training losses decomposes into the loss on each patch, the P1NN can be pre-trained patchwise. Each patch-speci\fc component of the P1NN is a fully connected MLP with 8 \u00028 inputs\nand 11 outputs with a softmax output layer. SMLP-hints uses the the standardization given\nin Equation 3 but with \u000f= 0.\nThe standardization is a crucial step for training the SMLP on the Pentomino dataset, and\nyields much sparser outputs, as seen on Figures 3 and 4. If the standardization is not used,"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f19b-6db4-11ee-b59e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-28", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "yields much sparser outputs, as seen on Figures 3 and 4. If the standardization is not used,\neven SMLP-hints could not solve the Pentomino task. In general, the standardization step\ndampens the small activations and augments larger ones(reducing the noise). Centering the\nactivations of each feature detector in a neural network has been studied in (Raiko et al., 2012)\nand (Vatanen et al., 2013). They proposed that transforming the outputs of each hidden neuron\nin a multi-layer perceptron network to have zero output and zero slope on average makes \frst\norder optimization methods closer to the second order techniques.\nBy default, the SMLP uses recti\fer hidden units as activation function, we found a signi\fcant boost by using recti\fcation compared to hyperbolic tangent and sigmoid activation\nfunctions. The P1NN has a highly overcomplete architecture with 1024 hidden units per patch,\nand L1 and L2 weight decay regularization coe\u000ecients on the weights (not the biases) are respectively 1e-6 and 1e-5. The learning rate for the P1NN is 0.75. 1 training epoch was enough"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f19c-6db4-11ee-a020-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-29", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "for the P1NN to learn the features of Pentomino shapes perfectly on the 40000 training examples. The P2NN has 2048 hidden units. L1 and L2 penalty coe\u000ecients for the P2NN are\n1e-6, and the learning rate is 0.1. These were selected by trial and error based on validation\nset error. Both P1NN (for each patch) and P2NN are fully-connected neural networks, even\nthough P1NN globally is a special kind of convolutional neural network.\nFilters of the \frst layer of SMLP are shown in Figure 6. These are the examples of the\n\flters obtained with the SLMP-hints trained with 40k examples, whose results are given in\nTable 1. Those \flters look very noisy but they work perfectly on the Pentomino task.\n3.2.3 Deep and Structured Supervised MLP without Hints (SMLP-nohints)\nSMLP-nohints uses the same connectivity pattern (and deep architecture) that is also used\nin the SMLP-hints architecture, but without using the intermediate targets ( Y). It directly\npredicts the \fnal outcome of the task ( Z), using the same number of hidden units, the same\n10"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f19d-6db4-11ee-b87f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-30", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "predicts the \fnal outcome of the task ( Z), using the same number of hidden units, the same\n10\nFigure 3: Bar chart of concatenated softmax output activations hoof P1NN (11\u000264=704 outputs) in SMLP-hints before standardization, for a selected example. There are very\nlarge spikes at each location for one of the possible 11 outcome (1 of K representation).\n11\nFigure 4: Softmax output activations hoof P1NN at SMLP-hints before standardization. There\nare positive spiked outputs at the locations where there is a Pentomino shape. Positive and negative spikes arise because most of the outputs are near an average value.\nActivations are higher at the locations where there is a pentomino shape.\n12\nStructured MLP \nArchitecture with \nHints\nFinal Binary task \nlabels\nIntermediate level \ntargets .\nSecond \nLevel \nNeural \nNetwork\nFirst \nLevel \nNeural \nNetwork\nFigure 5: Structured MLP architecture, used with hints (trained in two phases, \frst P1NN,\nbottom two layers, then P2NN, top two layers). In SMLP-hints, P1NN is trained"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f19e-6db4-11ee-90f0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-31", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "bottom two layers, then P2NN, top two layers). In SMLP-hints, P1NN is trained\non each 8x8 patch extracted from the image and the softmax output probabilities of\nall 64 patches are concatenated into a 64 \u000211 vector that forms the input of P2NN.\nOnlyUandVare learned in the P1NN and its output on each patch is fed into\nP2NN. The \frst level and the second level neural networks are trained separately,\nnot jointly.\nFigure 6: Filters of Structured MLP architecture, trained with hints on 40k examples.\n13\nconnectivity and the same activation function for the hidden units as SMLP-hints. 120 hyperparameter values have been evaluated by randomly selecting the number of hidden units\nfrom [64;128;256;512;1024;1200;2048] and randomly sampling 20 learning rates uniformly in\nthe log-domain within the interval of [0 :008;0:8]. Two fully connected hidden layers with 1024\nhidden units (same as P1NN) per patch is used and 2048 (same as P2NN) for the last hidden layer, with twenty training epochs. For this network the best results are obtained with a\nlearning rate of 0.05.7\nStructured MLP \nArchitecture without \nHints\nFinal Binary task"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f19f-6db4-11ee-bb0e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-32", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "learning rate of 0.05.7\nStructured MLP \nArchitecture without \nHints\nFinal Binary task \nlabels\nSecond \nLevel \nNeural \nNetwork\nFirst \nLevel \nNeural \nNetwork\nFigure 7: Structured MLP architecture, used without hints (SMLP-nohints). It is the same\narchitecture as SMLP-hints (Figure 5) but with both parts (P1NN and P2NN) trained\njointly with respect to the \fnal binary classi\fcation task.\nWe chose to experiment with various SMLP-nohint architectures and optimization procedures, trying unsuccessfully to achieve as good results with SMLP-nohint as with SMLP-hints.\nRecti\fer Non-Linearity A recti\fer nonlinearity is used for the activations of MLP hidden\nlayers. We observed that using piecewise linear nonlinearity activation function such as the\nrecti\fer can make the optimization more tractable.\n7. The source code of the structured MLP is available at the github repository: https://github.com/caglar/\nstructured_mlp\n14"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a0-6db4-11ee-b348-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-33", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "7. The source code of the structured MLP is available at the github repository: https://github.com/caglar/\nstructured_mlp\n14\nFigure 8: First layer \flters learned by the Structured MLP architecture, trained without using hints on 447600 examples with online SGD and a sigmoid intermediate layer\nactivation.\nIntermediate Layer The output of the P1NN is considered as an intermediate layer of the\nSMLP. For the SMLP-hints, only softmax output activations have been tried at the intermediate\nlayer, and that su\u000eced to learn the task. Since things did not work nearly as well with\nthe SMLP-nohints, several di\u000berent activation functions have been tried: softmax( \u0001), tanh(\u0001),\nsigmoid(\u0001) and linear activation functions.\nStandardization Layer Normalization at the last layer of the convolutional neural networks\nhas been used occasionaly to encourage the competition between the hidden units. (Jarrett\net al., 2009a) used a local contrast normalization layer in their architecture which performs\nsubtractive and divisive normalization. A local contrast normalization layer enforces a local\ncompetition between adjacent features in the feature map and between features at the same"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a1-6db4-11ee-b2ea-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-34", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "subtractive and divisive normalization. A local contrast normalization layer enforces a local\ncompetition between adjacent features in the feature map and between features at the same\nspatial location in di\u000berent feature maps. Similarly (Krizhevsky et al., 2012) observed that\nusing a local response layer that enjoys the bene\ft of using local normalization scheme aids\ngeneralization.\nStandardization has been observed to be crucial for both SMLP trained with or without hints. In both SMLP-hints and SMLP-nohints experiments, the neural network was not\nable to generalize or even learn the training set without using standardization in the SMLP\nintermediate layer, doing just chance performance. More speci\fcally, in the SMLP-nohints architecture, standardization is part of the computational graph, hence the gradients are being\nbackpropagated through it. The mean and the standard deviation is computed for each hidden\nunit separately at the intermediate layer as in Equation 4. But in order to prevent numerical\nunder\rows or over\rows during the backpropagation we have used \u000f= 1e\u00008 (Equation 3).\nThe bene\ft of having sparse activations may be speci\fcally important for the ill-conditioned"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a2-6db4-11ee-8ca8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-35", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "The bene\ft of having sparse activations may be speci\fcally important for the ill-conditioned\nproblems, for the following reasons. When a hidden unit is \\o\u000b\", its gradient (the derivative of\nthe loss with respect to its output) is usually close to 0 as well, as seen here. That means that all\no\u000b-diagonal second derivatives involving that hidden unit (e.g. its input weights) are also near 0.\nThis is basically like removing some columns and rows from the Hessian matrix associated with\na particular example. It has been observed that the condition number of the Hessian matrix\n(speci\fcally, its largest eigenvalue) increases as the size of the network increases (Dauphin and\nBengio, 2013), making training considerably slower and ine\u000ecient (Dauphin and Bengio, 2013).\nHence one would expect that as sparsity of the gradients (obtained because of sparsity of the\nactivations) increases, training would become more e\u000ecient, as if we were training a smaller\nsub-network for each example, with shared weights across examples, as in dropouts (Hinton\net al., 2012).\nIn Figure 9, the activation of each hidden unit in a bar chart is shown: the e\u000bect of\nstandardization is signi\fcant, making the activations sparser."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a3-6db4-11ee-a7ff-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-36", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "In Figure 9, the activation of each hidden unit in a bar chart is shown: the e\u000bect of\nstandardization is signi\fcant, making the activations sparser.\n15\n(a) Before standardization.\n (b) After standardization.\nFigure 9: Activations of the intermediate-level hidden units of an SLMP-nohints for a particular\nexamples (x-axis: hidden unit number, y-axis: activation value). Left (a): before\nstandardization. Right (b): after standardization.\nIn Figure 10, one can see the activation histogram of the SMLP-nohints intermediate layer,\nshowing the distribution of activation values, before and after standardization. Again the\nsparsifying e\u000bect of standardization is very apparent.\n(a) Before standardization.\n (b) After standardization.\nFigure 10: Distribution histogram of activation values of SMLP-nohints intermediate layer.\nLeft (a): before standardization. Right (b): after standardization.\nIn Figures 10 and 9, the intermediate level activations of SMLP-nohints are shown before\nand after standardization. These are for the same SMLP-nohints architecture whose results\nare presented on Table 1. For that same SMLP, the Adadelta (Zeiler, 2012) adaptive learning\n16"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a4-6db4-11ee-b6ab-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-37", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "are presented on Table 1. For that same SMLP, the Adadelta (Zeiler, 2012) adaptive learning\n16\nrate scheme has been used, with 512 hidden units for the hidden layer of P1NN and recti\fer\nactivation function. For the output of the P1NN, 11 sigmoidal units have been used while\nP2NN had 1200 hidden units with recti\fer activation function. The output nonlinearity of the\nP2NN is a sigmoid and the training objective is the binary crossentropy.\nAdaptive Learning Rates We have experimented with several di\u000berent adaptive learning\nrate algorithms. We tried rmsprop8, Adadelta (Zeiler, 2012), Adagrad (Duchi et al., 2010) and\na linearly (1/t) decaying learning rate (Bengio, 2013b). For the SMLP-nohints with sigmoid\nactivation function we have found Adadelta(Zeiler, 2012) converging faster to an e\u000bective local\nminima and usually yielding better generalization error compared to the others.\n3.2.4 Deep and Structured MLP with Unsupervised Pre-Training\nSeveral experiments have been conducted using an architecture similar to the SMLP-nohints,"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a5-6db4-11ee-990d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-38", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "3.2.4 Deep and Structured MLP with Unsupervised Pre-Training\nSeveral experiments have been conducted using an architecture similar to the SMLP-nohints,\nbut by using unsupervised pre-training of P1NN, with Denoising Auto-Encoder (DAE) and/or\nContractive Auto-Encoders (CAE). Supervised \fne-tuning proceeds as in the deep and structured MLP without hints. Because an unsupervised learner may not focus the representation\njust on the shapes, a larger number of intermediate-level units at the output of P1NN has been\nexplored: previous work on unsupervised pre-training generally found that larger hidden layers\nwere optimal when using unsupervised pre-training, because not all unsupervised features will\nbe relevant to the task at hand. Instead of limiting to 11 units per patch, we experimented\nwith networks with up to 20 hidden (i.e., code) units per patch in the second-layer patch-wise\nauto-encoder.\nIn Appendix 5.1 we also provided the result of some experiments with binary-binary RBMs\ntrained on 8\u00028 patches from the 40k training dataset.\nIn unsupervised pretraining experiments in this paper, both contractive auto-encoder (CAE)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a6-6db4-11ee-8b4b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-39", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "trained on 8\u00028 patches from the 40k training dataset.\nIn unsupervised pretraining experiments in this paper, both contractive auto-encoder (CAE)\nwith sigmoid nonlinearity and binary cross entropy cost function and denoising auto-encoder (DAE)\nhave been used. In the second layer, experiments were performed with a DAE with recti\fer\nhidden units utilizing L1 sparsity and weight decay on the weights of the auto-encoder. Greedy\nlayerwise unsupervised training procedure is used to train the deep auto-encoder architecture\n(Bengio et al., 2007). In unsupervised pretraining experiments, tied weights have been used.\nDi\u000berent combinations of CAE and DAE for unsupervised pretraining have been tested, but\nnone of the con\fgurations tested managed to learn the Pentomino task, as shown in Table 1.\n3.3 Experiments with 1 of K representation\nTo explore the e\u000bect of changing the complexity of the input representation on the di\u000eculty\nof the task, a set of experiments have been designed with symbolic representations of the\ninformation in each patch. In all cases an empty patch is represented with a 0 vector. These\nrepresentation can be seen as an alternative input for a P2NN-like network, i.e., they were fed"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a7-6db4-11ee-a7d7-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-40", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "representation can be seen as an alternative input for a P2NN-like network, i.e., they were fed\nas input to an MLP or another black-box classi\fer.\nThe following four experiments have been conducted, each one using one using a di\u000berent\ninput representation for each patch:\n8. This is learning rate scaling method that is discussed by G. Hinton in his Video Lecture 6.5 - rmsprop: Divide\nthe gradient by a running average of its recent magnitude. COURSERA: Neural Networks for Machine\nLearning, 2012.\n17\nAlgorithm 20k dataset 40k dataset 80k dataset\nTraining Test Training Test Training Test\nError Error Error Error Error Error\nSVM RBF 26.2 50.2 28.2 50.2 30.2 49.6\nK Nearest Neighbors 24.7 50.0 25.3 49.5 25.6 49.0\nDecision Tree 5.8 48.6 6.3 49.4 6.9 49.9\nRandomized Trees 3.2 49.8 3.4 50.5 3.5 49.1\nMLP 26.5 49.3 33.2 49.9 27.2 50.1\nConvnet/Lenet5 50.6 49.8 49.4 49.8 50.2 49.8"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a8-6db4-11ee-a7b9-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-41", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Convnet/Lenet5 50.6 49.8 49.4 49.8 50.2 49.8\nMaxout Convnet 14.5 49.5 0.0 50.1 0.0 44.6\n2 layer sDA 49.4 50.3 50.2 50.3 49.7 50.3\nStruct. Supervised MLP w/o hints 0.0 48.6 0.0 36.0 0.0 12.4\nStruct. MLP+CAE Supervised Finetuning 50.5 49.7 49.8 49.7 50.3 49.7\nStruct. MLP+CAE+DAE, Supervised Finetuning 49.1 49.7 49.4 49.7 50.1 49.7\nStruct. MLP+DAE+DAE, Supervised Finetuning 49.5 50.3 49.7 49.8 50.3 49.7\nStruct. MLP with Hints 0.21 30.7 0 3.1 0 0.01\nTable 1: The error percentages with di\u000berent learning algorithms on Pentomino dataset with\ndi\u000berent number of training examples."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1a9-6db4-11ee-a38d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-42", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Table 1: The error percentages with di\u000berent learning algorithms on Pentomino dataset with\ndi\u000berent number of training examples.\nExperiment 1-Onehot representation without transformations: In this experiment several trials have been done with a 10-input one-hot vector per patch. Each input corresponds to an object category given in clear, i.e., the ideal input for P2NN if a supervised\nP1NN perfectly did its job.\nExperiment 2-Disentangled representations: In this experiment, we did trials with 16\nbinary inputs per patch, 10 one-hot bits for representing each object category, 4 for\nrotations and 2 for scaling, i.e., the whole information about the input is given, but it is\nperfectly disentangled. This would be the ideal input for P2NN if an unsupervised P1NN\nperfectly did its job.\nExperiment 3-Onehot representation with transformations: For each of the ten object\ntypes there are 8 = 4 \u00022 possible transformations. Two objects in two di\u000berent patches\nare the considered \\the same\" (for the \fnal task) if their category is the same regardless\nof the transformations. The one-hot representation of a patch corresponds to the crossproduct between the 10 object shape classes and the 4 \u00022 transformations, i.e., one out of"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1aa-6db4-11ee-837c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-43", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "of the transformations. The one-hot representation of a patch corresponds to the crossproduct between the 10 object shape classes and the 4 \u00022 transformations, i.e., one out of\n80=10\u00024\u00022 possibilities represented in an 80-bit one-hot vector. This also contains all\nthe information about the input image patch, but spread out in a kind of non-parametric\nand non-informative (not disentangled) way, like a perfect memory-based unsupervised\nlearner (like clustering) could produce. Nevertheless, the shape class would be easier to\nread out from this representation than from the image representation (it would be an OR\nover 8 of the bits).\nExperiment 4-Onehot representation with 80 choices: This representation has the same\n1 of 80 one-hot representation per patch but the target task is de\fned di\u000berently. Two objects in two di\u000berent patches are considered the same i\u000b they have exactly the same 80-bit\nonehot representation (i.e., are of the same object category with the same transformation\napplied).\nThe \frst experiment is a sanity check. It was conducted with single hidden-layered MLP's\nwith recti\fer and tanh nonlinearity, and the task was learned perfectly (0 error on both training\nand test dataset) with very few training epochs.\n18"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ab-6db4-11ee-ab5e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-44", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "with recti\fer and tanh nonlinearity, and the task was learned perfectly (0 error on both training\nand test dataset) with very few training epochs.\n18\n0 100 200 300 400 500 600 700 800 9000.00.10.20.30.40.50.6\nTraining Error Rate\nTest Error Rate(a) Training and Test Errors for Experiment 4\n0 100 200 300 400 500 600 700 8000.00.10.20.30.40.50.6\nTraining Error Rate\nTest Error Rate (b) Training and Test Errors for Experiment 3\nFigure 11: Tanh MLP training curves. Left (a): The training and test errors of Experiment\n3 over 800 training epochs with 100k training examples using Tanh MLP. Right\n(b):The training and test errors of Experiment 4 over 700 training epochs with 100k\ntraining examples using Tanh MLP.\nThe results of Experiment 2 are given in Table 2. To improve results, we experimented\nwith the Maxout non-linearity in a feedforward MLP (Goodfellow et al., 2013) with two hidden\nlayers. Unlike the typical Maxout network mentioned in the original paper, regularizers have\nbeen deliberately avoided in order to focus on the optimization issue, i.e: no weight decay, norm\nconstraint on the weights, or dropout. Although learning from a disentangled representation is"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ac-6db4-11ee-bce8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-45", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "constraint on the weights, or dropout. Although learning from a disentangled representation is\nmore di\u000ecult than learning from perfect object detectors, it is feasible with some architectures\nsuch as the Maxout network. Note that this representation is the kind of representation that\none could hope an unsupervised learning algorithm could discover, at best, as argued in Bengio\net al. (2012).\nThe only results obtained on the validation set for Experiment 3 and Experiment 4 are\nshown respectively in Table 3 and Table 4. In these experiments a tanh MLP with two hidden\nlayers have been tested with the same hyperparameters. In experiment 3 the complexity of the\nproblem comes from the transformations (8=4 \u00022) and the number of object types.\nBut in experiment 4, the only source of complexity of the task comes from the number of\ndi\u000berent object types. These results are in between the complete failure and complete success\nobserved with other experiments, suggesting that the task could become solvable with better\ntraining or more training examples. Figure 11 illustrates the progress of training a tanh MLP,\non both the training and test error, for Experiments 3 and 4. Clearly, something has been\nlearned, but the task is not nailed yet. On experiment 3 for both maxout and tanh the maxout\nthere was a long plateau where the training error and objective stays almost same. Maxout"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ad-6db4-11ee-968e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-46", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "there was a long plateau where the training error and objective stays almost same. Maxout\ndid just chance on the experiment for about 120 iterations on the training and the test set.\nBut after 120th iteration the training and test error started decline and eventually it was able\nto solve the task. Moreover as seen from the curves in Figure 11(a) and 11(b), the training\nand test error curves are almost the same for both tasks. This implies that for onehot inputs,\nwhether you increase the number of possible transformations for each object or the number of\n19\nLearning Algorithm Training Error Test Error\nSVM 0.0 35.6\nRandom Forests 1.29 40.475\nTanh MLP 0.0 0.0\nMaxout MLP 0.0 0.0\nTable 2: Performance of di\u000berent learning algorithms on disentangled representation in Experiment 2.\nLearning Algorithm Training Error Test Error\nSVM 11.212 32.37\nRandom Forests 24.839 48.915\nTanh MLP 0.0 22.475\nMaxout MLP 0.0 0.0\nTable 3: Performance of di\u000berent learning algorithms using a dataset with onehot vector and\n80 inputs as discussed for Experiment 3.\nobject categories, as soon as the number of possible con\fgurations is same, the complexity of\nthe problem is almost the same for the MLP."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ae-6db4-11ee-bc51-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-47", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "object categories, as soon as the number of possible con\fgurations is same, the complexity of\nthe problem is almost the same for the MLP.\n3.4 Does the E\u000bect Persist with Larger Training Set Sizes?\nThe results shown in this section indicate that the problem in the Pentomino task clearly\nis not just a regularization problem, but rather basically hinges on an optimization problem.\nOtherwise, we would expect test error to decrease as the number of training examples increases.\nThis is shown \frst by studying the online case and then by studying the ordinary training\ncase with a \fxed size training set but considering increasing training set sizes. In the online\nminibatch setting, parameter updates are performed as follows:\n\u0012t+1=\u0012t\u0000\u0001\u0012t (5)\n\u0001\u0012t=\u000fPN\nir\u0012tL(xt;\u0012t)\nN(6)\nwhereL(xt;\u0012t) is the loss incurred on example xtwith parameters \u0012t, wheret2Z+and\u000f\nis the learning rate.\nOrdinary batch algorithms converge linearly to the optimum \u0012\u0003, however the noisy gradient\nestimates in the online SGD will cause parameter \u0012to \ructuate near the local optima. However,\nonline SGD directly optimizes the expected risk, because the examples are drawn iid from the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1af-6db4-11ee-bc5a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-48", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "online SGD directly optimizes the expected risk, because the examples are drawn iid from the\nground-truth distribution (Bottou, 2010). Thus:\nL1=E[L(x;\u0012)] =Z\nxL(x;\u0012)p(x)dx (7)\n20\nLearning Algorithm Training Error Test Error\nSVM 4.346 40.545\nRandom Forests 23.456 47.345\nTanh MLP 0 25.8\nTable 4: Performance of di\u000berent algorithms using a dataset with onehot vector and 80 binary\ninputs as discussed in Experiment 4.\nwhereL1is the generalization error. Therefore online SGD is trying to minimize the\nexpected risk with noisy updates. Those noisy updates have the e\u000bect of regularizer:\n\u0001\u0012t=\u000fPN\nir\u0012tL(xt;\u0012t)\nN=\u000fr\u0012tL(x;\u0012t) +\u000f\u0018t (8)\nwherer\u0012tL(x;\u0012t) is the true gradient and \u0018tis the zero-mean stochastic gradient \\noise\"\ndue to computing the gradient over a \fnite-size minibatch sample.\nWe would like to know if the problem with the Pentomino dataset is more a regularization"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b0-6db4-11ee-8bda-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-49", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "due to computing the gradient over a \fnite-size minibatch sample.\nWe would like to know if the problem with the Pentomino dataset is more a regularization\nor an optimization problem. An SMLP-nohints model was trained by online SGD with the randomly generated online Pentomino stream. The learning rate was adaptive, with the Adadelta\nprocedure (Zeiler, 2012) on minibatches of 100 examples. In the online SGD experiments, two\nSMLP-nohints that is trained with and without standardization at the intermediate layer with\nexactly the same hyperparameters are tested. The SMLP-nohints P1NN patch-wise submodel\nhas 2048 hidden units and the SMLP intermediate layer has 1152 = 64 \u000218 hidden units. The\nnonlinearity that is used for the intermediate layer is the sigmoid. P2NN has 2048 hidden units.\nSMLP-nohints has been trained either with or without standardization on top of the output\nunits of the P1NN. The experiments illustrated in Figures 12 and 13 are with the same SMLP\nwithout hints architecture for which results are given in Table 1. In those graphs only the results\nfor the training on the randomly generated 545400 Pentomino samples have been presented. As"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b1-6db4-11ee-89b4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-50", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "without hints architecture for which results are given in Table 1. In those graphs only the results\nfor the training on the randomly generated 545400 Pentomino samples have been presented. As\nshown in the plots SMLP-nohints was not able to generalize without standardization. Although\nwithout standardization the training loss seems to decrease initially, it eventually gets stuck in\na plateau where training loss doesn't change much.\nTraining of SMLP-nohints online minibatch SGD is performed using standardization in the\nintermediate layer and Adadelta learning rate adaptation, on 1046000 training examples from\nthe randomly generated Pentomino stream. At the end of the training, test error is down to\n27.5%, which is much better than chance but from from the score obtained with SMLP-hints\nof near 0 error.\nIn another SMLP-nohints experiment without standardization the model is trained with the\n1580000 Pentomino examples using online minibatch SGD. P1NN has 2048 hidden units and 16\nsigmoidal outputs per patch. for the P1NN hidden layer. P2NN has 1024 hidden units for the\nhidden layer. Adadelta is used to adapt the learning rate. At the end of training this SMLP,\nthe test error remained stuck, at 50.1%.\n21\n0 100 200 300 400 500 600 700 800"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b2-6db4-11ee-af06-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-51", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "the test error remained stuck, at 50.1%.\n21\n0 100 200 300 400 500 600 700 800\nBatch no0.200.250.300.350.400.450.500.55Test Error\nSMLP with standardization\nSMLP without standardizationFigure 12: Test errors of SMLP-nohints with and without standardization in the intermediate\nlayer. Sigmoid as an intermediate layer activation has been used. Each tick (batch\nno) in the x-axis represents 400 examples.\n22\n0 100 200 300 400 500 600 700\nBatch no01234567Training LossSMLP with standardization\nSMLP without standardizationFigure 13: Training errors of SMLP-nohints with and without standardization in the intermediate layer. Sigmoid nonlinearity has been used as an intermediate layer activation\nfunction. The x-axis is in units of blocks of 400 examples in the training set.\n23\n3.4.1 Experiments with Increased Training Set Size\nHere we consider the e\u000bect of training di\u000berent learners with di\u000berent numbers of training\nexamples. For the experimental results shown in Table 1, 3 training set sizes (20k, 40k and 80k\nexamples) had been used. Each dataset was generated with di\u000berent random seeds (so they"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b3-6db4-11ee-b922-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-52", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "examples) had been used. Each dataset was generated with di\u000berent random seeds (so they\ndo not overlap). Figure 14 also shows the error bars for an ordinary MLP with three hidden\nlayers, for a larger range of training set sizes, between 40k and 320k examples. The number of\ntraining epochs is 8 (more did not help), and there are three hidden layers with 2048 feature\ndetectors. The learning rate we used in our experiments is 0.01. The activation function of the\nMLP is a tanh nonlinearity, while the L1, L2 penalty coe\u000ecients are both 1e-6.\nTable 1 shows that, without guiding hints, none of the state-of-art learning algorithms could\nperform noticeably better than a random predictor on the test set. This shows the importance\nof intermediate hints introduced in the SMLP. The decision trees and SVMs can over\ft the\ntraining set but they could not generalize on the test set. Note that the numbers reported in the\ntable are for hyper-parameters selected based on validation set error, hence lower training errors\nare possible if avoiding all regularization and taking large enough models. On the training set,\nthe MLP with two large hidden layers (several thousands) could reach nearly 0% training error,\nbut still did not manage to achieve good test error."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b4-6db4-11ee-bdd9-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-53", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "the MLP with two large hidden layers (several thousands) could reach nearly 0% training error,\nbut still did not manage to achieve good test error.\nIn the experiment results shown in Figure 14, we evaluate the impact of adding more training\ndata for the fully-connected MLP. As mentioned before for these experiments we have used a\nMLP with three hidden layers where each layer has 2048 hidden units. The tanh( \u0001) activation\nfunction is used with 0.05 learning rate and minibatches of size 200.\nAs can be seen from the \fgure, adding more training examples did not help either training\nor test error (both are near 50%, with training error slightly lower and test error slightly\nhigher), reinforcing the hypothesis that the di\u000ecult encountered is one of optimization, not of\nregularization.\nFigure 14: Training and test error bar charts for a regular MLP with 3 hidden layers. There\nis no signi\fcant improvement on the generalization error of the MLP as the new\ntraining examples are introduced.\n24\n3.5 Experiments on E\u000bect of Initializing with Hints\nInitialization of the parameters in a neural network can have a big impact on the learning\nand generalization (Glorot and Bengio, 2010). Previously Erhan et al. (2010) showed that"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b5-6db4-11ee-b0ce-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-54", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "and generalization (Glorot and Bengio, 2010). Previously Erhan et al. (2010) showed that\ninitializing the parameters of a neural network with unsupervised pretraining guides the learning towards basins of attraction of local minima that provides better generalization from the\ntraining dataset. In this section we analyze the e\u000bect of initializing the SMLP with hints and\nthen continuing without hints at the rest of the training. For experimental analysis of hints\nbased initialization, SMLP is trained for 1 training epoch using the hints and for 60 epochs it\nis trained without hints on the 40k examples training set. We also compared the same architecture with the same hyperparameters, against to SMLP-nohints trained for 61 iterations on\nthe same dataset. After one iteration of hint-based training SMLP obtained 9% training error\nand 39% test error. Following the hint based training, SMLP is trained without hints for 60\nepochs, but at epoch 18, it already got 0% training and 0% test error. The hyperparameters\nfor this experiment and the experiment that the results shown for the SMLP-hints in Table 1\nare the same. The test results for initialization with and without hints are shown on Figure 15.\nThis \fgure suggests that initializing with hints can give the same generalization performance\nbut training takes longer."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b6-6db4-11ee-bc41-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-55", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "This \fgure suggests that initializing with hints can give the same generalization performance\nbut training takes longer.\nFigure 15: Plots showing the test error of SMLP with random initialization vs initializing with\nhint based training.\n3.5.1 Further Experiments on Optimization for Pentomino Dataset\nWith extensive hyperparameter optimization and using standardization in the intermediate\nlevel of the SMLP with softmax nonlinearity, SMLP-nohints was able to get 5.3% training and\n25\n6.7% test error on the 80k Pentomino training dataset. We used the 2050 hidden units for the\nhidden layer of P1NN and 11 softmax output per patch. For the P2NN, we used 1024 hidden\nunits with sigmoid and learning rate 0.1 without using any adaptive learning rate method. This\nSMLP uses a recti\fer nonlinearity for hidden layers of both P1NN and P2NN. Considering that\narchitecture uses softmax as the intermediate activation function of SMLP-nohints. It is very\nlikely that P1NN is trying to learn the presence of speci\fc Pentomino shape in a given patch.\nThis architecture has a very large capacity in the P1NN, that probably provides it enough"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b7-6db4-11ee-b33d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-56", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "This architecture has a very large capacity in the P1NN, that probably provides it enough\ncapacity to learn the presence of Pentomino shapes at each patch e\u000bortlessly.\nAn MLP with 2 hidden layers, each 1024 recti\fer units, was trained using LBFGS (the\nimplementation from the scipy.optimize library) on 40k training examples, with gradients computed on batches of 10000 examples at each iteration. However, after convergence of training,\nthe MLP was still doing chance on the test dataset.\nWe also observed that using linear units for the intermediate layer yields better generalization error without standardization compared to using activation functions such as sigmoid,\ntanh and RELU for the intermediate layer. SMLP-nohints was able to get 25% generalization error with linear units without standardization whereas all the other activation functions\nthat has been tested failed to generalize with the same number of training iterations without\nstandardization and hints. This suggests that using non-linear intermediate-level activation\nfunctions without standardization introduces an optimization di\u000eculty for the SMLP-nohints,\nmaybe because the intermediate level acts like a bottleneck in this architecture.\n4. Conclusion and Discussion\nIn this paper we have shown an example of task which seems almost impossible to solve by\nstandard black-box machine learning algorithms, but can be almost perfectly solved when"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b8-6db4-11ee-99ef-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-57", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "4. Conclusion and Discussion\nIn this paper we have shown an example of task which seems almost impossible to solve by\nstandard black-box machine learning algorithms, but can be almost perfectly solved when\none encourages a semantics for the intermediate-level representation that is guided by prior\nknowledge. The task has the particularity that it is de\fned by the composition of two nonlinear sub-tasks (object detection on one hand, and a non-linear logical operation similar to\nXOR on the other hand).\nWhat is interesting is that in the case of the neural network, we can compare two networks\nwith exactly the same architecture but a di\u000berent pre-training, one of which uses the known\nintermediate concepts to teach an intermediate representation to the network. With enough\ncapacity and training time they can over\ft but did not not capture the essence of the task, as\nseen by test set performance.\nWe know that a structured deep network can learn the task, if it is initialized in the right\nplace, and do it from very few training examples. Furthermore we have shown that if one\npre-trains SMLP with hints for only one epoch, it can nail the task. But the exactly same\narchitecture which started training from random initialization, failed to generalize.\nConsider the fact that even SMLP-nohints with standardization after being trained using"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1b9-6db4-11ee-afa8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-58", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "architecture which started training from random initialization, failed to generalize.\nConsider the fact that even SMLP-nohints with standardization after being trained using\nonline SGD on 1046000 generated examples and still gets 27.5% test error. This is an indication\nthat the problem is not a regularization problem but possibly an inability to \fnd a good e\u000bective\nlocal minima of generalization error .\nWhat we hypothesize is that for most initializations and architectures (in particular the\nfully-connected ones), although it is possible to \fnd a good e\u000bective local minimum of training\nerror when enough capacity is provided, it is di\u000ecult (without the proper initialization) to \fnd a\ngood local minimum of generalization error. On the other hand, when the network architecture\n26\nis constrained enough but still allows it to represent a good solution (such as the structured\nMLP of our experiments), it seems that the optimization problem can still be di\u000ecult and even\ntraining error remains stuck high if the standardization isn't used. Standardization obviously\nmakes the training objective of the SMLP easier to optimize and helps it to \fnd at least a\nbetter e\u000bective local minimum of training error . This \fnding suggests that by using speci\fc\narchitectural constraints and sometimes domain speci\fc knowledge about the problem, one can"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ba-6db4-11ee-b3f0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-59", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "architectural constraints and sometimes domain speci\fc knowledge about the problem, one can\nalleviate the optimization di\u000eculty that generic neural network architectures face.\nIt could be that the combination of the network architecture and training procedure produces a training dynamics that tends to yield into these minima that are poor from the point of\nview of generalization error, even when they manage to nail training error by providing enough\ncapacity. Of course, as the number of examples increases, we would expect this discrepancy to\ndecrease, but then the optimization problem could still make the task unfeasible in practice.\nNote however that our preliminary experiments with increasing the training set size (8-fold) for\nMLPs did not reveal signs of potential improvements in test error yet, as shown in Figure 14.\nEven using online training on 545400 Pentomino examples, the SMLP-nohints architecture was\nstill doing far from perfect in terms of generalization error (Figure 12).\nThese \fndings bring supporting evidence to the \\Guided Learning Hypothesis\" and \\Deeper\nHarder Hypothesis\" from Bengio (2013a): higher level abstractions, which are expressed by\ncomposing simpler concepts, are more di\u000ecult to learn (with the learner often getting in an\ne\u000bective local minimum ), but that di\u000eculty can be overcome if another agent provides hints"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1bb-6db4-11ee-9236-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-60", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "e\u000bective local minimum ), but that di\u000eculty can be overcome if another agent provides hints\nof the importance of learning other, intermediate-level abstractions which are relevant to the\ntask.\nMany interesting questions remain open. Would a network without any guiding hint eventually \fnd the solution with a enough training time and/or with alternate parametrizations?\nTo what extent is ill-conditioning a core issue? The results with LBFGS were disappointing but\nchanges in the architectures (such as standardization of the intermediate level) seem to make\ntraining much easier. Clearly, one can reach good solutions from an appropriate initialization,\npointing in the direction of an issue with local minima, but it may be that good solutions are\nalso reachable from other initializations, albeit going through a tortuous ill-conditioned path\nin parameter space. Why did our attempts at learning the intermediate concepts in an unsupervised way fail? Are these results speci\fc to the task we are testing or a limitation of the\nunsupervised feature learning algorithm tested? Trying with many more unsupervised variants and exploring explanatory hypotheses for the observed failures could help us answer that.\nFinally, and most ambitious, can we solve these kinds of problems if we allow a community\nof learners to collaborate and collectively discover and combine partial solutions in order to\nobtain solutions to more abstract tasks like the one presented here? Indeed, we would like to"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1bc-6db4-11ee-bf2c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-61", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "of learners to collaborate and collectively discover and combine partial solutions in order to\nobtain solutions to more abstract tasks like the one presented here? Indeed, we would like to\ndiscover learning algorithms that can solve such tasks without the use of prior knowledge as\nspeci\fc and strong as the one used in the SMLP here. These experiments could be inspired by\nand inform us about potential mechanisms for collective learning through cultural evolutions\nin human societies.\nAcknowledgments\nWe would like to thank to the ICLR 2013 reviewers for their insightful comments, and NSERC,\nCIFAR, Compute Canada and Canada Research Chairs for funding.\n27\nReferences\nA. Ben-Hur and J. Weston. A user's guide to support vector machines. Methods in Molecular\nBiology , 609:223{239, 2010.\nY. Bengio, P. Lamblin, D. Popovici, and H. Larochelle. Greedy layer-wise training of deep\nnetworks. In NIPS'2006 , 2007.\nYoshua Bengio. Learning deep architectures for AI. Foundations and Trends in Machine\nLearning , 2(1):1{127, 2009. Also published as a book. Now Publishers, 2009.\nYoshua Bengio. Evolving culture vs local minima. In Growing Adaptive Machines: Integrating"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1bd-6db4-11ee-b70e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-62", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Yoshua Bengio. Evolving culture vs local minima. In Growing Adaptive Machines: Integrating\nDevelopment and Learning in Arti\fcial Neural Networks , number also as ArXiv 1203.2990v1,\npages T. Kowaliw, N. Bredeche & R. Doursat, eds. Springer-Verlag, March 2013a. URL\nhttp://arxiv.org/abs/1203.2990 .\nYoshua Bengio. Practical recommendations for gradient-based training of deep architectures.\nIn K.-R. M\u007f uller, G. Montavon, and G. B. Orr, editors, Neural Networks: Tricks of the Trade .\nSpringer, 2013b.\nYoshua Bengio, Jerome Louradour, Ronan Collobert, and Jason Weston. Curriculum learning.\nIn L\u0013 eon Bottou and Michael Littman, editors, Proceedings of the Twenty-sixth International\nConference on Machine Learning (ICML'09) . ACM, 2009a.\nYoshua Bengio, Jerome Louradour, Ronan Collobert, and Jason Weston. Curriculum learning.\nInICML'09 , 2009b.\nYoshua Bengio, Aaron Courville, and Pascal Vincent. Unsupervised feature learning and deep"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1be-6db4-11ee-b44d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-63", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "InICML'09 , 2009b.\nYoshua Bengio, Aaron Courville, and Pascal Vincent. Unsupervised feature learning and deep\nlearning: A review and new perspectives. Technical Report arXiv:1206.5538, U. Montreal,\n2012. URL http://arxiv.org/abs/1206.5538 .\nYoshua Bengio, Aaron Courville, and Pascal Vincent. Unsupervised feature learning and deep\nlearning: A review and new perspectives. IEEE Trans. Pattern Analysis and Machine Intelligence (PAMI) , 2013.\nJames Bergstra, Olivier Breuleux, Fr\u0013 ed\u0013 eric Bastien, Pascal Lamblin, Razvan Pascanu, Guillaume Desjardins, Joseph Turian, David Warde-Farley, and Yoshua Bengio. Theano: a CPU\nand GPU math expression compiler. In Proceedings of the Python for Scienti\fc Computing\nConference (SciPy) , 2010.\nL\u0013 eon Bottou. Large-scale machine learning with stochastic gradient descent. In Proceedings of\nCOMPSTAT'2010 , pages 177{186. Springer, 2010.\nLeo Breiman. Random forests. Machine Learning , 45(1):5{32, 2001."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1bf-6db4-11ee-b5af-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-64", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "COMPSTAT'2010 , pages 177{186. Springer, 2010.\nLeo Breiman. Random forests. Machine Learning , 45(1):5{32, 2001.\nD. C. Ciresan, U. Meier, L. M. Gambardella, and J. Schmidhuber. Deep big simple neural nets\nfor handwritten digit recognition. Neural Computation , 22:1{14, 2010.\nYann Dauphin and Yoshua Bengio. Big neural networks waste capacity. Technical Report\narXiv:1301.3583, Universite de Montreal, 2013.\nRichard Dawkins. The Sel\fsh Gene . Oxford University Press, 1976.\n28\nJ. Duchi, E. Hazan, and Y. Singer. Adaptive subgradient methods for online learning and\nstochastic optimization. Journal of Machine Learning Research , 12:2121{2159, 2010.\nDumitru Erhan, Yoshua Bengio, Aaron Courville, Pierre-Antoine Manzagol, Pascal Vincent,\nand Samy Bengio. Why does unsupervised pre-training help deep learning? Journal of\nMachine Learning Research , 11:625{660, February 2010.\nFran\u0018 cois Fleuret, Ting Li, Charles Dubout, Emma K Wampler, Steven Yantis, and Donald"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c0-6db4-11ee-b502-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-65", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Fran\u0018 cois Fleuret, Ting Li, Charles Dubout, Emma K Wampler, Steven Yantis, and Donald\nGeman. Comparing machines and humans on a visual categorization test. Proceedings of the\nNational Academy of Sciences , 108(43):17621{17625, 2011.\nX. Glorot, A. Bordes, and Y. Bengio. Deep sparse recti\fer neural networks. In AISTATS ,\n2011a.\nXavier Glorot and Yoshua Bengio. Understanding the di\u000eculty of training deep feedforward\nneural networks. In JMLR W&CP: Proceedings of the Thirteenth International Conference\non Arti\fcial Intelligence and Statistics (AISTATS 2010) , volume 9, pages 249{256, May\n2010.\nXavier Glorot, Antoine Bordes, and Yoshua Bengio. Deep sparse recti\fer neural networks. In\nJMLR W&CP: Proceedings of the Fourteenth International Conference on Arti\fcial Intelligence and Statistics (AISTATS 2011) , April 2011b.\nIan J. Goodfellow, David Warde-Farley, Mehdi Mirza, Aaron Courville, and Yoshua Bengio.\nMaxout networks. In ICML , 2013."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c1-6db4-11ee-9491-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-66", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "Ian J. Goodfellow, David Warde-Farley, Mehdi Mirza, Aaron Courville, and Yoshua Bengio.\nMaxout networks. In ICML , 2013.\nMaciej Henneberg. Decrease of human skull size in the holocene. Human biology , pages 395{405,\n1988.\nMaciej Henneberg and Maryna Steyn. Trends in cranial capacity and cranial index in subsaharan africa during the holocene. American journal of human biology , 5(4):473{479, 1993.\nJ. Henrich and R. McElreath. The evolution of cultural evolution. Evolutionary Anthropology:\nIssues, News, and Reviews , 12(3):123{135, 2003.\nGeo\u000brey E. Hinton, Simon Osindero, and Yee Whye Teh. A fast learning algorithm for deep\nbelief nets. Neural Computation , 18:1527{1554, 2006.\nGeo\u000brey E. Hinton, Nitish Srivastava, Alex Krizhevsky, Ilya Sutskever, and Ruslan Salakhutdinov. Improving neural networks by preventing co-adaptation of feature detectors. Technical\nreport, arXiv:1207.0580, 2012."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c2-6db4-11ee-967b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-67", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "report, arXiv:1207.0580, 2012.\nC.W. Hsu, C.C. Chang, C.J. Lin, et al. A practical guide to support vector classi\fcation, 2003.\nKevin Jarrett, Koray Kavukcuoglu, Marc'Aurelio Ranzato, and Yann LeCun. What is the\nbest multi-stage architecture for object recognition? In Proc. International Conference on\nComputer Vision (ICCV'09) , pages 2146{2153. IEEE, 2009a.\nKevin Jarrett, Koray Kavukcuoglu, Marc'Aurelio Ranzato, and Yann LeCun. What is the best\nmulti-stage architecture for object recognition? In ICCV'09 , 2009b.\n29\nFaisal Khan, Xiaojin Zhu, and Bilge Mutlu. How do humans teach: On curriculum learning and\nteaching dimension. In Advances in Neural Information Processing Systems 24 (NIPS'11) ,\npages 1449{1457, 2011.\nAlex Krizhevsky, Ilya Sutskever, and Geo\u000brey Hinton. ImageNet classi\fcation with deep\nconvolutional neural networks. In Advances in Neural Information Processing Systems 25\n(NIPS'2012) . 2012."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c3-6db4-11ee-8eac-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-68", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "convolutional neural networks. In Advances in Neural Information Processing Systems 25\n(NIPS'2012) . 2012.\nKai A. Krueger and Peter Dayan. Flexible shaping: how learning in small steps helps. Cognition ,\n110:380{394, 2009.\nG. Kunapuli, K.P. Bennett, R. Maclin, and J.W. Shavlik. The adviceptron: Giving advice to\nthe perceptron. Proceedings of the Conference on Arti\fcial Neural Networks In Engineering\n(ANNIE 2010) , 2010.\nHugo Larochelle, Yoshua Bengio, Jerome Louradour, and Pascal Lamblin. Exploring strategies\nfor training deep neural networks. Journal of Machine Learning Research , 10:1{40, 2009.\nY. LeCun, L. Bottou, Y. Bengio, and P. Ha\u000bner. Gradient-based learning applied to document\nrecognition. Proceedings of the IEEE , 86(11):2278{2324, 1998.\nT.M. Mitchell. The need for biases in learning generalizations . Department of Computer\nScience, Laboratory for Computer Science Research, Rutgers Univ., 1980.\nT.M. Mitchell and S.B. Thrun. Explanation-based neural network learning for robot control.\nAdvances in Neural information processing systems , pages 287{287, 1993."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c4-6db4-11ee-a16a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-69", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "T.M. Mitchell and S.B. Thrun. Explanation-based neural network learning for robot control.\nAdvances in Neural information processing systems , pages 287{287, 1993.\nR. Montague. Universal grammar. Theoria , 36(3):373{398, 1970.\nV. Nair and G. E Hinton. Recti\fed linear units improve restricted Boltzmann machines. In\nICML'10 , 2010.\nL.B.J.H.F.R.A. Olshen and C.J. Stone. Classi\fcation and regression trees. Belmont, Calif.:\nWadsworth , 1984.\nJoseph O'Sullivan. Integrating initialization bias and search bias in neural network learning,\n1996.\nF. Pedregosa, G. Varoquaux, A. Gramfort, V. Michel, B. Thirion, O. Grisel, M. Blondel,\nP. Prettenhofer, R. Weiss, V. Dubourg, et al. Scikit-learn: Machine learning in python. The\nJournal of Machine Learning Research , 12:2825{2830, 2011.\nGail B. Peterson. A day of great illumination: B. F. Skinner's discovery of shaping. Journal of\nthe Experimental Analysis of Behavior , 82(3):317{328, 2004."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c5-6db4-11ee-be59-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-70", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "the Experimental Analysis of Behavior , 82(3):317{328, 2004.\nTapani Raiko, Harri Valpola, and Yann LeCun. Deep learning made easier by linear transformations in perceptrons. In International Conference on Arti\fcial Intelligence and Statistics ,\npages 924{932, 2012.\nSalah Rifai, Pascal Vincent, Xavier Muller, Xavier Glorot, and Yoshua Bengio. Contractive\nauto-encoders: Explicit invariance during feature extraction. In ICML'2011 , 2011.\n30\nSalah Rifai, Yoshua Bengio, Yann Dauphin, and Pascal Vincent. A generative process for sampling contractive auto-encoders. In Proceedings of the Twenty-nine International Conference\non Machine Learning (ICML'12) . ACM, 2012. URL http://icml.cc/discuss/2012/590.\nhtml .\nR. Salakhutdinov and G.E. Hinton. Deep Boltzmann machines. In Proceedings of the Twelfth\nInternational Conference on Arti\fcial Intelligence and Statistics (AISTATS 2009) , volume 8,\n2009.\nBurrhus F. Skinner. Reinforcement today. American Psychologist , 13:94{99, 1958."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c6-6db4-11ee-a238-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-71", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "2009.\nBurrhus F. Skinner. Reinforcement today. American Psychologist , 13:94{99, 1958.\nR.J. Solomono\u000b. A system for incremental learning based on algorithmic probability. In Proceedings of the Sixth Israeli Conference on Arti\fcial Intelligence, Computer Vision and Pattern\nRecognition , pages 515{527. Citeseer, 1989.\nG.G. Towell and J.W. Shavlik. Knowledge-based arti\fcial neural networks. Arti\fcial intelligence , 70(1):119{165, 1994.\nTommi Vatanen, Tapani Raiko, Harri Valpola, and Yann LeCun. Pushing stochastic gradient\ntowards second-order methods{backpropagation learning with transformations in nonlinearities. arXiv preprint arXiv:1301.3476 , 2013.\nPascal Vincent, Hugo Larochelle, Isabelle Lajoie, Yoshua Bengio, and Pierre-Antoine Manzagol.\nStacked denoising autoencoders: Learning useful representations in a deep network with a\nlocal denoising criterion. Journal of Machine Learning Research , 11:3371{3408, December\n2010.\nLuis Von Ahn, Manuel Blum, Nicholas J Hopper, and John Langford. Captcha: Using hard"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c7-6db4-11ee-bd56-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-72", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "2010.\nLuis Von Ahn, Manuel Blum, Nicholas J Hopper, and John Langford. Captcha: Using hard\nai problems for security. In Advances in CryptologyEUROCRYPT 2003 , pages 294{311.\nSpringer, 2003.\nJason Weston, Fr\u0013 ed\u0013 eric Ratle, and Ronan Collobert. Deep learning via semi-supervised embedding. In William W. Cohen, Andrew McCallum, and Sam T. Roweis, editors, Proceedings of the Twenty-\ffth International Conference on Machine Learning (ICML'08) ,\npages 1168{1175, New York, NY, USA, 2008. ACM. ISBN 978-1-60558-205-4. doi:\n10.1145/1390156.1390303.\nMatthew D Zeiler. Adadelta: An adaptive learning rate method. arXiv preprint\narXiv:1212.5701 , 2012.\n5. Appendix\n5.1 Binary-Binary RBMs on Pentomino Dataset\nWe trained binary-binary RBMs (both visible and hidden are binary) on 8 \u00028 patches extracted\nfrom the Pentomino Dataset using PCD (stochastic maximum likelihood), a weight decay of\n.0001 and a sparsity penalty9. We used 256 hidden units and trained by SGD with a batch"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c8-6db4-11ee-ac77-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-73", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": ".0001 and a sparsity penalty9. We used 256 hidden units and trained by SGD with a batch\nsize of 32 and a annealing learning rate (Bengio, 2013b) starting from 1e-3 with annealing rate\n9. implemented as TorontoSparsity in pylearn2, see the yaml \fle in the repository for more details\n31\n1.000015. The RBM is trained with momentum starting from 0.5. The biases are initialized to\n-2 in order to get a sparse representation. The RBM is trained for 120 epochs (approximately\n50 million updates).\nAfter pretraining the RBM, its parameters are used to initialize the \frst layer of an SMLPnohints network. As in the usual architecture of the SMLP-nohints on top of P1NN, there is\nan intermediate layer. Both P1NN and the intermediate layer have a sigmoid nonlinearity, and\nthe intermediate layer has 11 units per location. This SMLP-nohints is trained with Adadelta\nand standardization at the intermediate layer10.\n0 5 10 15 20 25 30 35 40 45\nEpoch0.00.10.20.30.40.5Error percentageRBM Test and Training Errors\nTraining Error\nTest Error"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1c9-6db4-11ee-808f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-74", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "0 5 10 15 20 25 30 35 40 45\nEpoch0.00.10.20.30.40.5Error percentageRBM Test and Training Errors\nTraining Error\nTest Error\nFigure 16: Training and test errors of an SMLP-nohints network whose \frst layer is pre-trained\nas an RBM. Training error reduces to 0% at epoch 42, but test error is still chance.\n5.2 Experimental Setup and Hyper-parameters\n5.2.1 Decision Trees\nWe used the decision tree implementation in the scikit-learn (Pedregosa et al., 2011) python\npackage which is an implementation of the CART (Regression Trees) algorithm. The CART\nalgorithm constructs the decision tree recursively and partitions the input space such that the\nsamples belonging to the same category are grouped together (Olshen and Stone, 1984). We\nused The Gini index as the impurity criteria. We evaluated the hyper-parameter con\fgurations with a grid-search. We cross-validated the maximum depth ( maxdepth ) of the tree (for\npreventing the algorithm to severely over\ft the training set) and minimum number of samples\n10. In our auto-encoder experiments we directly fed features to P2NN without standardization and Adadelta.\n32"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ca-6db4-11ee-a607-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-75", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "10. In our auto-encoder experiments we directly fed features to P2NN without standardization and Adadelta.\n32\nFigure 17: Filters learned by the binary-binary RBM after training on the 40k examples. The\nRBM did learn the edge structure of Pentomino shapes.\nFigure 18: 100 samples generated from trained RBM. All the generated samples are valid Pentomino shapes.\n33\nrequired to create a split ( minsplit). 20 di\u000berent con\fgurations of hyper-parameter values were\nevaluated. We obtained the best validation error with maxdepth = 300 andminsplit = 8.\n5.2.2 Support Vector Machines\nWe used the \\Support Vector Classi\fer (SVC)\" implementation from the scikit-learn package\nwhich in turn uses the libsvm's Support Vector Machine (SVM) implementation. Kernelbased SVMs are non-parametric models that map the data into a high dimensional space and\nseparate di\u000berent classes with hyperplane(s) such that the support vectors for each category\nwill be separated by a large margin. We cross-validated three hyper-parameters of the model\nusing grid-search: C,\rand the type of kernel( kerneltype).Cis the penalty term (weight"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1cb-6db4-11ee-85cf-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-76", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "using grid-search: C,\rand the type of kernel( kerneltype).Cis the penalty term (weight\ndecay) for the SVM and \ris a hyper-parameter that controls the width of the Gaussian for\nthe RBF kernel. For the polynomial kernel, \rcontrols the \rexibility of the classi\fer (degree\nof the polynomial) as the number of parameters increases (Hsu et al., 2003; Ben-Hur and\nWeston, 2010). We evaluated forty-two hyper-parameter con\fgurations. That includes, two\nkernel types:fRBF; Polynomial g; three gammas:f1e\u00002;1e\u00003;1e\u00004gfor the RBF kernel,\nf1;2;5gfor the polynomial kernel, and seven Cvalues among:f0:1;1;2;4;8;10;16g. As a\nresult of the grid search and cross-validation, we have obtained the best test error by using the\nRBF kernel, with C= 2 and\r= 1.\n5.2.3 Multi Layer Perceptron\nWe have our own implementation of Multi Layer Perceptron based on the Theano (Bergstra\net al., 2010) machine learning libraries. We have selected 2 hidden layers, the recti\fer activation"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1cc-6db4-11ee-8a82-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-77", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "et al., 2010) machine learning libraries. We have selected 2 hidden layers, the recti\fer activation\nfunction, and 2048 hidden units per layer. We cross-validated three hyper-parameters of the\nmodel using random-search, sampling the learning rates \u000fin log-domain, and selecting L1\nandL2 regularization penalty coe\u000ecients in sets of \fxed values, evaluating 64 hyperparameter\nvalues. The range of the hyperparameter values are \u000f2[0:0001;1],L12f0:;1e\u00006;1e\u00005;1e\u00004g\nandL22f0;1e\u00006;1e\u00005g. As a result, the following were selected: L1 = 1e\u00006,L2 = 1e\u00005\nand\u000f= 0:05.\n5.2.4 Random Forests\nWe used scikit-learn's implementation of \\Random Forests\" decision tree learning. The Random Forests algorithm creates an ensemble of decision trees by randomly selecting for each tree\na subset of features and applying bagging to combine the individual decision trees (Breiman,\n2001). We have used grid-search and cross-validated the maxdepth ,minsplit, and number"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1cd-6db4-11ee-a735-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-78", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "2001). We have used grid-search and cross-validated the maxdepth ,minsplit, and number\nof trees (nestimators ). We have done the grid-search on the following hyperparameter values,nestimators2 f5;10;15;25;50g,maxdepth2 f100;300;600;900g, andminsplits2\nf1;4;16g. We obtained the best validation error with maxdepth = 300,minsplit = 4 and\nnestimators = 10.\n5.2.5 k-Nearest Neighbors\nWe used scikit-learn's implementation of k-Nearest Neighbors (k-NN). k-NN is an instancebased, lazy learning algorithm that selects the training examples closest in Euclidean distance\nto the input query. It assigns a class label to the test example based on the categories of the\nkclosest neighbors. The hyper-parameters we have evaluated in the cross-validation are the\nnumber of neighbors ( k) andweights . Theweights hyper-parameter can be either \\uniform\" or\n34\n\\distance\". With \\uniform\", the value assigned to the query point is computed by the majority\nvote of the nearest neighbors. With \\distance\", each value assigned to the query point is\ncomputed by weighted majority votes where the weights are computed with the inverse distance"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ce-6db4-11ee-bcfb-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-79", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "vote of the nearest neighbors. With \\distance\", each value assigned to the query point is\ncomputed by weighted majority votes where the weights are computed with the inverse distance\nbetween the query point and the neighbors. We have used nneighbours2f1;2;4;6;8;12gand\nweights2f\"uniform \";\"distance \"gfor hyper-parameter search. As a result of cross-validation\nand grid search, we obtained the best validation error with k= 2 andweights =\\uniform\".\n5.2.6 Convolutional Neural Nets\nWe used a Theano (Bergstra et al., 2010) implementation of Convolutional Neural Networks\n(CNN) from the deep learning tutorial at deeplearning.net , which is based on a vanilla\nversion of a CNN LeCun et al. (1998). Our CNN has two convolutional layers. Following\neach convolutional layer, we have a max-pooling layer. On top of the convolution-poolingconvolution-pooling layers there is an MLP with one hidden layer. In the cross-validation we\nhave sampled 36 learning rates in log-domain in the range [0 :0001;1] and the number of \flters"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1cf-6db4-11ee-b96f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-80", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "have sampled 36 learning rates in log-domain in the range [0 :0001;1] and the number of \flters\nfrom the range [10 ;20;30;40;50;60] uniformly. For the \frst convolutional layer we used 9 \u00029\nreceptive \felds in order to guarantee that each object \fts inside the receptive \feld. As a result\nof random hyperparameter search and doing manual hyperparameter search on the validation\ndataset, the following values were selected:\n\u000fThe number of features used for the \frst layer is 30 and the second layer is 60.\n\u000fFor the second convolutional layer, 7 \u00027 receptive \felds. The stride for both convolutional\nlayers is 1.\n\u000fConvolved images are downsampled by a factor of 2 \u00022 at each pooling operation.\n\u000fThe learning rate for CNN is 0.01 and it was trained for 8 epochs.\n5.2.7 Maxout Convolutional Neural Nets\nWe used the pylearn2 ( https://github.com/lisa-lab/pylearn2 ) implementation of maxout\nconvolutional networks (Goodfellow et al., 2013). There are two convolutional layers in the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d0-6db4-11ee-bb68-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-81", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "convolutional networks (Goodfellow et al., 2013). There are two convolutional layers in the\nselected architecture, without any pooling. In the last convolutional layer, there is a maxout\nnon-linearity. The following were selected by cross-validation: learning rate, number of channels\nfor the both convolution layers, number of kernels for the second layer and number of units and\npieces per maxout unit in the last layer, a linearly decaying learning rate, momentum starting\nfrom 0.5 and saturating to 0.8 at the 200'th epoch. Random search for the hyperparameters\nwas used to evaluate 48 di\u000berent hyperparameter con\fgurations on the validation dataset. For\nthe \frst convolutional layer, 8 \u00028 kernels were selected to make sure that each Pentomino shape\n\fts into the kernel. Early stopping was used and test error on the model that has the best\nvalidation error is reported. Using norm constraint on the fan-in of the \fnal softmax units\nyields slightly better result on the validation dataset.\nAs a result of cross-validation and manually tuning the hyperparameters we used the following hyperparameters:\n\u000f16 channels per convolutional layer. 600 hidden units for the maxout layer.\n\u000f6x6 kernels for the second convolutional layer.\n35"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d1-6db4-11ee-a42a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-82", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "\u000f16 channels per convolutional layer. 600 hidden units for the maxout layer.\n\u000f6x6 kernels for the second convolutional layer.\n35\n\u000f5 pieces for the convolution layers and 4 pieces for the maxout layer per maxout units.\n\u000fWe decayed the learning rate by the factor of 0.001 and the initial learning rate is 0.026367.\nBut we scaled the learning rate of the second convolutional layer by a constant factor of\n0.6.\n\u000fThe norm constraint (on the incoming weights of each unit) is 1.9365.\nFigure 19 shows the \frst layer \flters of the maxout convolutional net, after being trained\non the 80k training set for 85 epochs.\nFigure 19: Maxout convolutional net \frst layer \flters. Most of the \flters were able to learn\nthe basic edge structure of the Pentomino shapes.\n5.2.8 Stacked Denoising Auto-Encoders\nDenoising Auto-Encoders (DAE) are a form of regularized auto-encoder (Bengio et al., 2013).\nThe DAE forces the hidden layer to discover more robust features and prevents it from simply\nlearning the identity by reconstructing the input from a corrupted version of it (Vincent et al.,"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d2-6db4-11ee-9c6d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-83", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "The DAE forces the hidden layer to discover more robust features and prevents it from simply\nlearning the identity by reconstructing the input from a corrupted version of it (Vincent et al.,\n2010). Two DAEs were stacked, resulting in an unsupervised transformation with two hidden\nlayers of 1024 units each. Parameters of all layers are then \fne-tuned with supervised \fnetuning using logistic regression as the classi\fer and SGD as the gradient-based optimization\nalgorithm. The stochastic corruption process is binomial (0 or 1 replacing each input value,\nwith probability 0.2). The selected learning rate is \u000f0= 0:01 for the DAe and \u000f1= 0:1 for\nsupervised \fne-tuning. Both L1 and L2 penalty for the DAEs and for the logistic regression\nlayer are set to 1e-6.\nCAE+MLP with Supervised Finetuning: A regularized auto-encoder which sometimes\noutperforms the DAE is the Contractive Auto-Encoder (CAE), (Rifai et al., 2012), which\npenalizes the Frobenius norm of the Jacobian matrix of derivatives of the hidden units with\nrespect to the CAE's inputs. The CAE serves as pre-training for an MLP, and in the supervised"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d3-6db4-11ee-bcf4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-84", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "respect to the CAE's inputs. The CAE serves as pre-training for an MLP, and in the supervised\n\fne-tuning state, the Adagrad method was used to automatically tune the learning rate (Duchi\net al., 2010).\nAfter training a CAE with 100 sigmoidal units patch-wise, the features extracted on each\npatch are concatenated and fed as input to an MLP. The selected Jacobian penalty coe\u000ecient\nis 2, the learning rate for pre-training is 0.082 with batch size of 200 and 200 epochs of unsupervised learning are performed on the training set. For supervised \fnetuning, the learning\nrate is 0.12 over 100 epochs, L1 and L2 regularization penalty terms respectively are 1e-4 and\n1e-6, and the top-level MLP has 6400 hidden units.\n36\nGreedy Layerwise CAE+DAE Supervised Finetuning: For this experiment we stack a\nCAE with sigmoid non-linearities and then a DAE with recti\fer non-linearities during the pretraining phase. As recommended by Glorot et al. (2011b) we have used a softplus nonlinearity\nfor reconstruction, softplus (x) =log(1 +ex). We used an L1 penalty on the recti\fer outputs"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d4-6db4-11ee-9d03-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-85", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "for reconstruction, softplus (x) =log(1 +ex). We used an L1 penalty on the recti\fer outputs\nto obtain a sparser representation with recti\fer non-linearity and L2 regularization to keep the\nnon-zero weights small.\nThe main di\u000berence between the DAE and CAE is that the DAE yields more robust reconstruction whereas the CAE obtains more robust features (Rifai et al., 2011).\nAs seen on Figure 7 the weights U and V are shared on each patch and we concatenate the\noutputs of the last auto-encoder on each patch to feed it as an input to an MLP with a large\nhidden layer.\nWe used 400 hidden units for the CAE and 100 hidden units for DAE. The learning rate\nused for the CAE is 0.82 and for DAE it is 9*1e-3. The corruption level for the DAE (binomial\nnoise) is 0.25 and the contraction level for the CAE is 2.0. The L1 regularization penalty for\nthe DAE is 2.25*1e-4 and the L2 penalty is 9.5*1e-5. For the supervised \fnetuning phase the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d5-6db4-11ee-8efe-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-86", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "the DAE is 2.25*1e-4 and the L2 penalty is 9.5*1e-5. For the supervised \fnetuning phase the\nlearning rate used is 4*1e-4 with L1 and L2 penalties respectively 1e-5 and 1e-6. The top-level\nMLP has 6400 hidden units. The auto-encoders are each trained for 150 epochs while the whole\nMLP is \fne-tuned for 50 epochs.\nGreedy Layerwise DAE+DAE Supervised Finetuning: For this architecture, we have\ntrained two layers of denoising auto-encoders greedily and performed supervised \fnetuning\nafter unsupervised pre-training. The motivation for using two denoising auto-encoders is the\nfact that recti\fer nonlinearities work well with the deep networks but it is di\u000ecult to train\nCAEs with the recti\fer non-linearity. We have used the same type of denoising auto-encoder\nthat is used for the greedy layerwise CAE+DAE supervised \fnetuning experiment.\nIn this experiment we have used 400 hidden units for the \frst layer DAE and 100 hidden\nunits for the second layer DAE. The other hyperparameters for DAE and supervised \fnetuning"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d6-6db4-11ee-a681-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1301.4083-87", "title": "Knowledge Matters: Importance of Prior Information for Optimization", "source": "http://arxiv.org/pdf/1301.4083", "start_index": 0, "chroma:document": "units for the second layer DAE. The other hyperparameters for DAE and supervised \fnetuning\nare the same as with the CAE+DAE MLP Supervised Finetuning experiment.\n37"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d7-6db4-11ee-9cd1-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-0", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "DeepPose: Human Pose Estimation via Deep Neural Networks\nAlexander Toshev Christian Szegedy\nGoogle\n1600 Amphitheatre Pkwy\nMountain View, CA 94043\ntoshev,szegedy@google.com\nFigure 1. Besides extreme variability in articulations, many of the\njoints are barely visible. We can guess the location of the right\narm in the left image only because we see the rest of the pose and\nanticipate the motion or activity of the person. Similarly, the left\nbody half of the person on the right is not visible at all. These\nare examples of the need for holistic reasoning . We believe that\nDNNs can naturally provide such type of reasoning.\nAbstract\nWe propose a method for human pose estimation based\non Deep Neural Networks (DNNs). The pose estimation\nis formulated as a DNN-based regression problem towards\nbody joints. We present a cascade of such DNN regressors which results in high precision pose estimates. The\napproach has the advantage of reasoning about pose in a\nholistic fashion and has a simple but yet powerful formulation which capitalizes on recent advances in Deep Learning. We present a detailed empirical analysis with state-ofart or better performance on four academic benchmarks of\ndiverse real-world images.\n1. Introduction\nThe problem of human pose estimation, de\ufb01ned as the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d8-6db4-11ee-a661-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-1", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "diverse real-world images.\n1. Introduction\nThe problem of human pose estimation, de\ufb01ned as the\nproblem of localization of human joints, has enjoyed substantial attention in the computer vision community. In\nFig. 1, one can see some of the challenges of this problem \u2013 strong articulations, small and barely visible joints,\nocclusions and the need to capture the context.\nThe main stream of work in this \ufb01eld has been motivatedmainly by the \ufb01rst challenge, the need to search in the large\nspace of all possible articulated poses. Part-based models\nlend themselves naturally to model articulations ([16, 8])\nand in the recent years a variety of models with ef\ufb01cient\ninference have been proposed ([6, 19]).\nThe above ef\ufb01ciency, however, is achieved at the cost of\nlimited expressiveness \u2013 the use of local detectors, which\nreason in many cases about a single part, and most importantly by modeling only a small subset of all interactions\nbetween body parts. These limitations, as exempli\ufb01ed in\nFig. 1, have been recognized and methods reasoning about\npose in a holistic manner have been proposed [15, 21] but\nwith limited success in real-world problems.\nIn this work we ascribe to this holistic view of human\npose estimation. We capitalize on recent developments of"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1d9-6db4-11ee-8a29-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-2", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "with limited success in real-world problems.\nIn this work we ascribe to this holistic view of human\npose estimation. We capitalize on recent developments of\ndeep learning and propose a novel algorithm based on a\nDeep Neural Network (DNN). DNNs have shown outstanding performance on visual classi\ufb01cation tasks [14] and more\nrecently on object localization [23, 9]. However, the question of applying DNNs for precise localization of articulated\nobjects has largely remained unanswered. In this paper we\nattempt to cast a light on this question and present a simple\nand yet powerful formulation of holistic human pose estimation as a DNN.\nWe formulate the pose estimation as a joint regression\nproblem and show how to successfully cast it in DNN settings. The location of each body joint is regressed to using\nas an input the full image and a 7-layered generic convolutional DNN. There are two advantages of this formulation.\nFirst, the DNN is capable of capturing the full context of\neach body joint \u2013 each joint regressor uses the full image\nas a signal. Second, the approach is substantially simpler\nto formulate than methods based on graphical models \u2013 no\nneed to explicitly design feature representations and detectors for parts; no need to explicitly design a model topology\nand interactions between joints. Instead, we show that a\ngeneric convolutional DNN can be learned for this problem."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1da-6db4-11ee-a595-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-3", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "and interactions between joints. Instead, we show that a\ngeneric convolutional DNN can be learned for this problem.\nFurther, we propose a cascade of DNN-based pose predictors. Such a cascade allows for increased precision of\n1arXiv:1312.4659v3  [cs.CV]  20 Aug 2014\njoint localization. Starting with an initial pose estimation,\nbased on the full image, we learn DNN-based regressors\nwhich re\ufb01nes the joint predictions by using higher resolution sub-images.\nWe show state-of-art results or better than state-of-art on\nfour widely used benchmarks against all reported results.\nWe show that our approach performs well on images of people which exhibit strong variation in appearance as well as\narticulations. Finally, we show generalization performance\nby cross-dataset evaluation.\n2. Related Work\nThe idea of representing articulated objects in general,\nand human pose in particular, as a graph of parts has been\nadvocated from the early days of computer vision [16]. The\nso called Pictorial Strictures (PSs), introduced by Fishler\nand Elschlager [8], were made tractable and practical by\nFelzenszwalb and Huttenlocher [6] using the distance transform trick. As a result, a wide variety of PS-based models"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1db-6db4-11ee-8a93-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-4", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Felzenszwalb and Huttenlocher [6] using the distance transform trick. As a result, a wide variety of PS-based models\nwith practical signi\ufb01cance were subsequently developed.\nThe above tractability, however, comes with the limitation of having a tree-based pose models with simple binary\npotential not depending on image data. As a result, research\nhas focused on enriching the representational power of the\nmodels while maintaining tractability. Earlier attempts to\nachieve this were based on richer part detectors [19, 1, 4].\nMore recently, a wide variety of models expressing complex\njoint relationships were proposed. Yang and Ramanan [27]\nuse a mixture model of parts. Mixture models on the full\nmodel scale, by having mixture of PSs, have been studied\nby Johnson and Everingham [13]. Richer higher-order spatial relationships were captured in a hierarchical model by\nTian et al. [25]. A different approach to capture higherorder relationship is through image-dependent PS models,\nwhich can be estimated via a global classi\ufb01er [26, 20, 18].\nApproaches which ascribe to our philosophy of reasoning about pose in a holistic manner have shown limited\npracticality. Mori and Malik [15] try to \ufb01nd for each test\nimage the closest exemplar from a set of labeled images"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1dc-6db4-11ee-9c6a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-5", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "practicality. Mori and Malik [15] try to \ufb01nd for each test\nimage the closest exemplar from a set of labeled images\nand transfer the joint locations. A similar nearest neighbor\nsetup is employed by Shakhnarovich et al. [21], who however use locality sensitive hashing. More recently, Gkioxari\net al. [10] propose a semi-global classi\ufb01er for part con\ufb01guration. This formulation has shown very good results on\nreal-world data, however, it is based on linear classi\ufb01ers\nwith less expressive representation than ours and is tested\non arms only. Finally, the idea of pose regression has been\nemployed by Ionescu et al. [11], however they reason about\n3D pose.\nThe closest work to ours uses convolution NNs together\nwith Neighborhood Component Analysis to regress toward\na point in an embedding representing pose [24]. However,\nthis work does not employ a cascade of networks. Cascadesof DNN regressors have been used for localization, however\nof facial points [22]. On the related problem of face pose\nestimation, Osadchy et al. [17] employ a NN-based pose\nembedding trained with a contrastive loss.\n3. Deep Learning Model for Pose Estimation"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1dd-6db4-11ee-91a4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-6", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "estimation, Osadchy et al. [17] employ a NN-based pose\nembedding trained with a contrastive loss.\n3. Deep Learning Model for Pose Estimation\nWe use the following notation. To express a pose, we encode the locations of all kbody joints in pose vector de\ufb01ned\nasy= (:::;yT\ni;:::)T;i2f1;:::;kg, where yicontains\nthexandycoordinates of the ithjoint. A labeled image is\ndenoted by (x;y)wherexstands for the image data and y\nis the ground truth pose vector.\nFurther, since the joint coordinates are in absolute image\ncoordinates, it proves bene\ufb01cial to normalize them w. r. t. a\nboxbbounding the human body or parts of it. In a trivial\ncase, the box can denote the full image. Such a box is de\ufb01ned by its center bc2R2as well as width bwand height\nbh:b= (bc;bw;bh). Then the joint yican be translated by\nthe box center and scaled by the box size which we refer to\nas normalization by b:\nN(yi;b) =\u00121=bw 0\n0 1=bh\u0013\n(yi\u0000bc) (1)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1de-6db4-11ee-bb8b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-7", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "as normalization by b:\nN(yi;b) =\u00121=bw 0\n0 1=bh\u0013\n(yi\u0000bc) (1)\nFurther, we can apply the same normalization to the elements of pose vector N(y;b) = (:::;N (yi;b)T;:::)Tresulting in a normalized pose vector . Finally, with a slight\nabuse of notation, we use N(x;b)to denote a crop of the\nimagexby the bounding box b, which de facto normalizes\nthe image by the box. For brevity we denote by N(\u0001)normalization with bbeing the full image box.\n3.1. Pose Estimation as DNN-based Regression\nIn this work, we treat the problem of pose estimation as\nregression, where the we train and use a function  (x;\u0012)2\nR2kwhich for an image xregresses to a normalized pose\nvector, where \u0012denotes the parameters of the model. Thus,\nusing the normalization transformation from Eq. (1) the\npose prediction y\u0003in absolute image coordinates reads\ny\u0003=N\u00001( (N(x);\u0012)) (2)\nDespite its simple formulation, the power and complexity of the method is in  , which is based on a convolutional"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1df-6db4-11ee-b733-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-8", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Despite its simple formulation, the power and complexity of the method is in  , which is based on a convolutional\nDeep Neural Network (DNN). Such a convolutional network consists of several layers \u2013 each being a linear transformation followed by a non-linear one. The \ufb01rst layer takes\nas input an image of prede\ufb01ned size and has a size equal to\nthe number of pixels times three color channels. The last\nlayer outputs the target values of the regression, in our case\n2kjoint coordinates.\nWe base the architecture of the  on the work by\nKrizhevsky et al. [14] for image classi\ufb01cation since it has\nshown outstanding results on object localization as well\n(xi, yi)\n(x(s-1)i, y (s-1) i)xsi - x(s-1)iysi - y(s-1)iInitial stageStage s\nsend re\ufb01ned values to next stage220 x 220\nDNN-based regressor\n27 x 27 x 128\n13 x 13 x 192\n13 x 13 x192\n13 x 13 x192\n4096 \n4096 \n55 x 55 x 48xiyi...DNN-based re\ufb01ner\n27 x 27 x 128\n13 x 13 x 192\n13 x 13 x192"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e0-6db4-11ee-9f55-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-9", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "4096 \n55 x 55 x 48xiyi...DNN-based re\ufb01ner\n27 x 27 x 128\n13 x 13 x 192\n13 x 13 x192\n13 x 13 x192\n4096 \n4096 \n55 x 55 x 48Figure 2. Left: schematic view of the DNN-based pose regression. We visualize the network layers with their corresponding dimensions,\nwhere convolutional layers are in blue, while fully connected ones are in green. We do not show the parameter free layers. Right: at stage\ns, a re\ufb01ning regressor is applied on a sub image to re\ufb01ne a prediction from the previous stage.\n[23]. In a nutshell, the network consists of 7layers (see\nFig. 2 left). Denote by Ca convolutional layer, by LRN\na local response normalization layer, Pa pooling layer\nand byFa fully connected layer. Only CandFlayers\ncontain learnable parameters, while the rest are parameter free. Both CandFlayers consist of a linear transformation followed by a nonlinear one, which in our case\nis a recti\ufb01ed linear unit. For Clayers, the size is de\ufb01ned as width\u0002height\u0002depth, where the \ufb01rst two dimensions have a spatial meaning while the depth de\ufb01nes"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e1-6db4-11ee-9297-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-10", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "the number of \ufb01lters. If we write the size of each layer in\nparentheses, then the network can be described concisely\nasC(55\u000255\u000296)\u0000LRN\u0000P\u0000C(27\u000227\u0002256)\u0000\nLRN\u0000P\u0000C(13\u000213\u0002384)\u0000C(13\u000213\u0002384)\u0000\nC(13\u000213\u0002256)\u0000P\u0000F(4096)\u0000F(4096) . The \ufb01lter\nsize for the \ufb01rst two Clayers is 11\u000211and5\u00025and for\nthe remaining three is 3\u00023. Pooling is applied after three\nlayers and contributes to increased performance despite the\nreduction of resolution. The input to the net is an image\nof220\u0002220which via stride of 4is fed into the network.\nThe total number of parameters in the above model is about\n40M. For further details, we refer the reader to [14].\nThe use of a generic DNN architecture is motivated by\nits outstanding results on both classi\ufb01cation and localization\nproblems. In the experimental section we show that such a\ngeneric architecture can be used to learn a model resulting\nin state-of-art or better performance on pose estimation as\nwell. Further, such a model is a truly holistic one \u2014 the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e2-6db4-11ee-8228-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-11", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "generic architecture can be used to learn a model resulting\nin state-of-art or better performance on pose estimation as\nwell. Further, such a model is a truly holistic one \u2014 the\n\ufb01nal joint location estimate is based on a complex nonlinear\ntransformation of the full image.\nAdditionally, the use of a DNN obviates the need to design a domain speci\ufb01c pose model. Instead such a model\nand the features are learned from the data. Although the regression loss does not model explicit interactions between\njoints, such are implicitly captured by all of the 7hidden\nlayers \u2013 all the internal features are shared by all joint regressors.\nTraining The difference to [14] is the loss. Instead of a\nclassi\ufb01cation loss, we train a linear regression on top of thelast network layer to predict a pose vector by minimizing\nL2distance between the prediction and the true pose vector. Since the ground truth pose vector is de\ufb01ned in absolute image coordinates and poses vary in size from image to\nimage, we normalize our training set Dusing the normalization from Eq. (1):\nDN=f(N(x);N(y))j(x;y)2Dg (3)\nThen theL2loss for obtaining optimal network parameters\nreads:\narg min\n\u0012X"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e3-6db4-11ee-a5cb-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-12", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Then theL2loss for obtaining optimal network parameters\nreads:\narg min\n\u0012X\n(x;y)2DNkX\ni=1jjyi\u0000 i(x;\u0012)jj2\n2 (4)\nFor clarity we write out the optimization over individual\njoints. It should be noted, that the above objective can\nbe used even if for some images not all joints are labeled.\nIn this case, the corresponding terms in the sum would be\nomitted.\nThe above parameters \u0012are optimized for using Backpropagation in a distributed online implementation. For\neach mini-batch of size 128, adaptive gradient updates are\ncomputed [3]. The learning rate, as the most important parameter, is set to 0:0005 . Since the model has large number\nof parameters and the used datasets are of relatively small\nsize, we augment the data using large number of randomly\ntranslated image crops (see Sec. 3.2), left/right \ufb02ips as well\nas DropOut regularization for the Flayers set to 0:6.\n3.2. Cascade of Pose Regressors\nThe pose formulation from the previous section has the\nadvantage that the joint estimation is based on the full image and thus relies on context. However, due to its \ufb01xed\ninput size of 220\u0002220, the network has limited capacity"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e4-6db4-11ee-a800-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-13", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "input size of 220\u0002220, the network has limited capacity\nto look at detail \u2013 it learns \ufb01lters capturing pose properties\nat coarse scale. These are necessary to estimate rough pose\nbut insuf\ufb01cient to always precisely localize the body joints.\nNote that we cannot easily increase the input size since this\nwill increase the already large number of parameters. In order to achieve better precision, we propose to train a cascade\nof pose regressors. At the \ufb01rst stage, the cascade starts off\nby estimating an initial pose as outlined in the previous section. At subsequent stages, additional DNN regressors are\ntrained to predict a displacement of the joint locations from\nprevious stage to the true location. Thus, each subsequent\nstage can be thought of as a re\ufb01nement of the currently predicted pose, as shown in Fig. 2.\nFurther, each subsequent stage uses the predicted joint\nlocations to focus on the relevant parts of the image \u2013 subimages are cropped around the predicted joint location from\nprevious stage and the pose displacement regressor for this\njoint is applied on this sub-image. In this way, subsequent\npose regressors see higher resolution images and thus learn\nfeatures for \ufb01ner scales which ultimately leads to higher\nprecision.\nWe use the same network architecture for all stages of\nthe cascade but learn different network parameters. For"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e5-6db4-11ee-9c20-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-14", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "features for \ufb01ner scales which ultimately leads to higher\nprecision.\nWe use the same network architecture for all stages of\nthe cascade but learn different network parameters. For\nstages2 f1;:::;Sgof totalScascade stages, we denote by\u0012sthe learned network parameters. Thus, the\npose displacement regressor reads  (x;\u0012s). To re\ufb01ne a\ngiven joint location yiwe will consider a joint bounding\nboxbicapturing the sub-image around yi:bi(y;\u001b) =\n(yi;\u001bdiam(y);\u001bdiam(y))having as center the i-th joint\nand as dimension the pose diameter scaled by \u001b. The diameter diam (y)of the pose is de\ufb01ned as the distance between\nopposing joints on the human torso, such as left shoulder\nand right hip, and depends on the concrete pose de\ufb01nition\nand dataset.\nUsing the above notation, at the stage s= 1we start with\na bounding box b0which either encloses the full image or\nis obtained by a person detector. We obtain an initial pose:\nStage 1 : y1 N\u00001( (N(x;b0);\u00121);b0)(5)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e6-6db4-11ee-b5f6-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-15", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "is obtained by a person detector. We obtain an initial pose:\nStage 1 : y1 N\u00001( (N(x;b0);\u00121);b0)(5)\nAt each subsequent stage s\u00152, for all joints i2f1;:::;kg\nwe regress \ufb01rst towards a re\ufb01nement displacement ys\ni\u0000\ny(s\u00001)\ni by applying a regressor on the sub image de\ufb01ned\nbyb(s\u00001)\ni from previous stage (s\u00001). Then, we estimate\nnew joint boxes bs\ni:\nStages:ys\ni y(s\u00001)\ni +N\u00001( i(N(x;b);\u0012s);b)(6)\nforb=b(s\u00001)\ni\nbs\ni (ys\ni;\u001bdiam(ys);\u001bdiam(ys)) (7)\nWe apply the cascade for a \ufb01xed number of stages S,\nwhich is determined as explained in Sec. 4.1.\nTraining The network parameters \u00121are trained as\noutlined in Sec. 3.1, Eq. (4). At subsequent stages"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e7-6db4-11ee-b9d4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-16", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Training The network parameters \u00121are trained as\noutlined in Sec. 3.1, Eq. (4). At subsequent stages\ns\u00152, the training is done identically with one important difference. Each joint ifrom a training example(x;y)is normalized using a different bounding box(y(s\u00001)\ni;\u001bdiam(y(s\u00001));\u001bdiam(y(s\u00001)))\u2013 the one centered at the prediction for the same joint obtained from previous stage \u2013 so that we condition the training of the stage\nbased on the model from previous stage.\nSince deep learning methods have large capacity, we\naugment the training data by using multiple normalizations\nfor each image and joint. Instead of using the prediction\nfrom previous stage only, we generate simulated predictions. This is done by randomly displacing the ground truth\nlocation for joint iby a vector sampled at random from a\n2-dimensional Normal distribution N(s\u00001)\ni with mean and\nvariance equal to the mean and variance of the observed displacements (y(s\u00001)\ni\u0000yi)across all examples in the training data. The full augmented training data can be de\ufb01ned\nby \ufb01rst sampling an example and a joint from the original\ndata at uniform and then generating a simulated prediction"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e8-6db4-11ee-aabd-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-17", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "by \ufb01rst sampling an example and a joint from the original\ndata at uniform and then generating a simulated prediction\nbased on a sampled displacement \u000efromN(s\u00001)\ni :\nDs\nA=f(N(x;b);N(yi;b))j\n(x;yi)\u0018D;\u000e\u0018N(s\u00001)\ni;\nb= (yi+\u000e;\u001bdiam(y))g\nThe training objective for cascade stage sis done as in\nEq. (4) by taking extra care to use the correct normalization\nfor each joint:\n\u0012s= arg min\n\u0012X\n(x;yi)2Ds\nAjjyi\u0000 i(x;\u0012)jj2\n2 (8)\n4. Empirical Evaluation\n4.1. Setup\nDatasets There is a wide variety of benchmarks for human pose estimation. In this work we use datasets, which\nhave large number of training examples suf\ufb01cient to train a\nlarge model such as the proposed DNN, as well as are realistic and challenging.\nThe \ufb01rst dataset we use is Frames Labeled In Cinema\n(FLIC), introduced by [20], which consists of 4000 training and 1000 test images obtained from popular Hollywood\nmovies. The images contain people in diverse poses and especially diverse clothing. For each labeled human, 10upper"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1e9-6db4-11ee-a890-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-18", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "movies. The images contain people in diverse poses and especially diverse clothing. For each labeled human, 10upper\nbody joints are labeled.\nThe second dataset we use is Leeds Sports Dataset [12]\nand its extension [13], which we will jointly denote by LSP.\nCombined they contain 11000 training and 1000 testing images. These are images from sports activities and as such\nare quite challenging in terms of appearance and especially\narticulations. In addition, the majority of people have 150\npixel height which makes the pose estimation even more\nchallenging. In this dataset, for each person the full body is\nlabeled with total 14joints.\nFor all of the above datasets, we de\ufb01ne the diameter of a\nposeyto be the distance between a shoulder and hip from\nopposing sides and denote it by diam (y). It should be noted,\nthat the joints in all datasets are arranged in a tree kinematically mimicking the human body. This allows for a de\ufb01nition of a limb being a pair of neighboring joints in the pose\ntree.\nMetrics In order to be able to compare with published results we will use two widely accepted evaluation metrics.\nPercentage of Correct Parts (PCP) measures detection rate\nof limbs, where a limb is considered detected if the distance\nbetween the two predicted joint locations and the true limb"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ea-6db4-11ee-a8fe-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-19", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Percentage of Correct Parts (PCP) measures detection rate\nof limbs, where a limb is considered detected if the distance\nbetween the two predicted joint locations and the true limb\njoint locations is at most half of the limb length [5]. PCP\nwas the initially preferred metric for evaluation, however it\nhas the drawback of penalizing shorter limbs, such as lower\narms, which are usually harder to detect.\nTo address this drawback, recently detection rates of\njoints are being reported using a different detection criterion \u2013 a joint is considered detected if the distance between\nthe predicted and the true joint is within a certain fraction of\nthe torso diameter. By varying this fraction, detection rates\nare obtained for varying degrees of localization precision.\nThis metric alleviates the drawback of PCP since the detection criteria for all joints are based on the same distance\nthreshold. We refer to this metric as Percent of Detected\nJoints (PDJ).\nExperimental Details For all the experiments we use the\nsame network architecture. Inspired by [7], we use a body\ndetector on FLIC to obtain initially a rough estimate of the\nhuman body bounding box. It is based on a face detector \u2013\nthe detected face rectangle is enlarged by a \ufb01xed scaler. This\nscaler is determined on the training data such that it contains\nall labeled joints. This face-based body detector results in"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1eb-6db4-11ee-8bd4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-20", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "scaler is determined on the training data such that it contains\nall labeled joints. This face-based body detector results in\na rough estimate, which however presents a good starting\npoint for our approach. For LSP we use the full image as\ninitial bounding box since the humans are relatively tightly\ncropped by design.\nUsing a small held-out set of 50 images for both datasets\nto determine the algorithm hyperparameters. To measure\noptimality of the parameters we used average over PDJ at\n0:2across all joints. The scaler \u001b, which de\ufb01nes the size of\nthe re\ufb01nement joint bounding box as a fraction of the pose\nsize, is determined as follows: for FLIC we chose \u001b= 1:0\nafter exploring values f0:8;1:0;1:2g, for LSP we use \u001b=\n2:0after tryingf1:5;1:7;2:0;2:3g. The number of cascade\nstagesSis determined by training stages until the algorithm\nstopped improving on the held-out set. For both FLIC and\nLSP we arrived at S= 3.\nTo improve generalization, for each cascade stage starting ats= 2 we augment the training data by sampling 40\nrandomly translated crop boxes for each joint as explained"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ec-6db4-11ee-8754-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-21", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "To improve generalization, for each cascade stage starting ats= 2 we augment the training data by sampling 40\nrandomly translated crop boxes for each joint as explained\nin Sec. 3.2. Thus, for LSP with 14joints and after mirroring the images and sampling the number training examples\nis11000\u000240\u00022\u000214 = 12M, which is essential for\ntraining a large network as ours.The presented algorithm allows for an ef\ufb01cient implementation. The running time is approx. 0:1sper image,\nas measured on a 12core CPU. This compares favorably\nto other approaches, as some of the current state-of-art approaches have higher complexity: [20] runs in approx. 4s,\nwhile [27] runs in 1:5s. The training complexity, however,\nis higher. The initial stage was trained within 3days on\napprox. 100 workers, most of the \ufb01nal performance was\nachieved after 12hours though. Each re\ufb01nement stage was\ntrained for 7days since the amount of data was 40\u0002larger\nthan the one for the initial stage due to the data augmentation in Sec. 3.2. Note that using more data led to increased\nperformance.\n4.2. Results and Discussion"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ed-6db4-11ee-a09f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-22", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "than the one for the initial stage due to the data augmentation in Sec. 3.2. Note that using more data led to increased\nperformance.\n4.2. Results and Discussion\nComparisons We present comparative results to other approaches. We compare on LSP using PCP metric in Fig. 1.\nWe show results for the four most challenging limbs \u2013 lower\nand upper arms and legs \u2013 as well as the average value\nacross these limbs for all compared algorithms. We clearly\noutperform all other approaches, especially achieving better estimation for legs. For example, for upper legs we obtain0:78up from 0:74for the next best performing method.\nIt is worth noting that while the other approaches exhibit\nstrengths for particular limbs, none of the other dataset consistently dominates across all limbs. In contrary, DeepPose\nshows strong results for all challenging limbs.\nUsing the PDJ metric allows us to vary the threshold for\nthe distance between prediction and ground truth, which de\ufb01nes a detection. This threshold can be thought of as a\nlocalization precision at which detection rates are plotted.\nThus one could compare approaches across different desired precisions. We present results on FLIC in Fig. 3 comparing against additional four methods as well is on LSP in\nFig. 4. For each dataset we train and test according the protocol for each dataset. Similarly to previous experiment we"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ee-6db4-11ee-8eba-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-23", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Fig. 4. For each dataset we train and test according the protocol for each dataset. Similarly to previous experiment we\noutperform all \ufb01ve algorithms. Our gains are bigger in the\nlow precision domain, in the cases where we detect rough\npose without precisely localizing the joints. On FLIC, at\nnormalized distance 0:2we obtain a an increase of detection\nrates by 0:15and0:2for elbow and wrists against the next\nbest performing method. On LSP, at normalized distance\n0:5we get an absolute increase of 0:1. At low precision\nregime of normalized distance of 0:2for LSP we show comparable performance for legs and slightly worse arms. This\ncan be attributed to the fact that the DNN-based approach\ncomputes joint coordinates using 7 layers of transformation,\nsome of which contain max pooling.\nAnother observation is that our approach works well for\nboth appearance heavy movie data as well as string articulation such as the sports images in LSP.\nEffects of cascade-based re\ufb01nement A single DNNbased joint regressor gives rough joint location. However,\n0 0.05 0.1 0.15 0.20.10.30.50.70.9Elbows\nNormalized distance to true jointDetection rate"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ef-6db4-11ee-afea-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-24", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "0 0.05 0.1 0.15 0.20.10.30.50.70.9Elbows\nNormalized distance to true jointDetection rate\n  \nDeepPoseMODECEichner et al.Yang et al.Sapp et al.\n0 0.05 0.1 0.15 0.20.10.30.50.70.9Wrists\nNormalized distance to true jointDetection rateFigure 3. Percentage of detected joints (PDJ) on FLIC for two joints: elbow and wrist. We compare DeepPose, after two cascade stages,\nwith four other approaches.\nMethodArm LegAve.Upper Lower Upper Lower\nDeepPose-st1 0.5 0.27 0.74 0.65 0.54\nDeepPose-st2 0.56 0.36 0.78 0.70 0.60\nDeepPose-st3 0.56 0.38 0.77 0.71 0.61\nDantone et al. [2] 0.45 0.25 0.65 0.61 0.49\nTian et al. [25]\u00030.52 0.33 0.70 0.60 0.56\nJohnson et al. [13] 0.54 0.38 0.75 0.66 0.58"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f0-6db4-11ee-8b11-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-25", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Johnson et al. [13] 0.54 0.38 0.75 0.66 0.58\nWang et al. [26]\u00030.565 0.37 0.76 0.68 0.59\nPishchulin [18]\u000e0.49 0.32 0.74 0.70 0.56\nTable 1. Percentage of Correct Parts (PCP) at 0.5 on LSP for DeepPose as well as \ufb01ve state-of-art approaches.\u0003The authors use a\nslightly looser version of PCP in which for each limb the average\ndistance from predicted limb joints belonging to the corresponding true joints is used to determine whether the limb is correctly\ndetected.\u000eThe authors use person-centric joint annotations.\n0 0.1 0.2 0.3 0.4 0.50.10.30.50.70.9Arms\nNormalized distance to true jointDetection rate\n  \nDeepPose \u2212 wristsDeepPose \u2212 elbowsDantone et al. \u2212 wristsDantone et al. \u2212 elbows\n0 0.1 0.2 0.3 0.4 0.50.10.30.50.70.9Legs\nNormalized distance to true jointDetection rate\n  \nDeepPose \u2212 ankleDeepPose \u2212 kneeDantone et al. \u2212 ankleDantone et al. \u2212 knee"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f1-6db4-11ee-83b0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-26", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Normalized distance to true jointDetection rate\n  \nDeepPose \u2212 ankleDeepPose \u2212 kneeDantone et al. \u2212 ankleDantone et al. \u2212 knee\nFigure 4. Percentage of detected joints (PDJ) on LSP for four\nlimbs for DeepPose and Dantone et al. [2] over an extended range\nof distances to true joint: [0;0:5]of the torso diameter. Results of\nDeepPose are plotted with solid lines while all the results by [2]\nare plotted in dashed lines. Results for the same joint from both\nalgorithms are colored with same color.\nto obtain higher precision the subsequent stages of the cascade, which serve as a re\ufb01nement of the initial prediction,\nare of paramount importance. To see this, in Fig. 5 we\npresent the joint detections at different precisions for the initial prediction as well as two subsequent cascade stages. As\nexpected, we can see that the major gains of the re\ufb01nement\n00.05 0.10.15 0.20.25 0.30.35 0.40.10.30.50.70.9Wrists\nNormalized distance to true jointDetection rate\n  \nDeepPose \u2212 initial stage 1DeepPose \u2212 stage 2DeepPose \u2212 stage 3"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f2-6db4-11ee-950e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-27", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Normalized distance to true jointDetection rate\n  \nDeepPose \u2212 initial stage 1DeepPose \u2212 stage 2DeepPose \u2212 stage 3\n0 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.40.10.30.50.70.9Elbows\nNormalized distance to true jointDetection rateFigure 5. Percent of detected joints (PDJ) on FLIC or the \ufb01rst three\nstages of the DNN cascade. We present results over larger spectrum of normalized distances between prediction and ground truth.\nprocedure are at high-precision regime of at normalized distances of [0:15;0:2]. Further, the major gains are achieved\nafter one stage of re\ufb01nement. The reason being that subsequent stages end up using smaller sub-images around each\njoint. And although the subsequent stages look at higher\nresolution inputs, they have more limited context.\nExamples of cases, where re\ufb01nement helps, are visualized in Fig. 6. The initial stage is usually successful at estimating a roughly correct pose, however, this pose is not\n\u201dsnapped\u201d to the correct one. For example, in row three the\npose has the right shape but incorrect scale. In the second\nrow, the predicted pose is translated north from the ideal"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f3-6db4-11ee-aaf8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-28", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "pose has the right shape but incorrect scale. In the second\nrow, the predicted pose is translated north from the ideal\none. In most cases, the second stage of the cascade resolves\nthis snapping problem and better aligns the joints. In more\nrare cases, such as in \ufb01rst row, further facade stages improve\non individual joints.\nCross-dataset Generalization To evaluate the generalization properties of our algorithm, we used the trained\nmodels on LSP and FLIC on two related datasets. The fullbody model trained on LSP is tested on the test portion of\nthe Image Parse dataset [19] with results presented in Table 2. The ImageParse dataset is similar to LSP as it contains people doing sports, however it contains a lot of people from personal photo collections involved in other activities. Further, the upper-body model trained on FLIC was\napplied on the whole Buffy dataset [7]. We can see that our\nInitial stage 1stage 2stage 3Figure 6. Predicted poses in red and ground truth poses in green\nfor the \ufb01rst three stages of a cascade for three examples.\n0 0.05 0.1 0.15 0.20.10.30.50.70.9Elbows\nNormalized distance to true jointDetection rate\n  \nEichner et al.Yang et al.Sapp et al.MODECDeepPose"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f4-6db4-11ee-a223-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-29", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Normalized distance to true jointDetection rate\n  \nEichner et al.Yang et al.Sapp et al.MODECDeepPose\n0 0.05 0.1 0.15 0.20.10.30.50.70.9Wrists\nNormalized distance to true jointDetection rate\nFigure 7. Percentage of detected joints (PDJ) on Buffy dataset\nfor two joints: elbow and wrist. The models have been trained on\nFLIC. We compare DeepPose, after two cascade stages, with four\nother approaches.\nMethodArm LegAve.Upper Lower Upper Lower\nDeepPose 0.8 0.75 0.71 0.5 0.69\nPishchulin [18] 0.80 0.70 0.59 037 0.62\nJohnson et al. [13] 0.75 0.67 0.67 0.46 0.64\nYang et al. [27] 0.69 0.64 0.55 0.35 0.56\nTable 2. Percentage of Correct Parts (PCP) at 0.5 on Image Parse\ndataset for DeepPose as well as two state-of-art approaches on Image Parse dataset. Results obtained from [18].\napproach can retain state-of-art performance compared to\nother approaches. This shows good generalization abilities."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f5-6db4-11ee-8f0f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-30", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "approach can retain state-of-art performance compared to\nother approaches. This shows good generalization abilities.\nExample poses To get a better idea of the performance of\nour algorithm, we visualize a sample of estimated poses on\nimages from LSP in Fig. 8. We can see that our algorithm is\nable to get correct pose for most of the joints under variety\nof conditions: upside-down people (row 1, column 1), severe foreshortening (row1, column 3), unusual poses (row\n3, column 5), occluded limbs as the occluded arms in row\n3, columns 2 and 6, unusual illumination conditions (row 3,\ncolumn 3). In most of the cases, when the estimated pose isnot precise, it still has a correct shape. For example, in the\nlast row some of the predicted limbs are not aligned with\nthe true locations, however the overall shape of the pose is\ncorrect. A common failure mode is confusing left with right\nside when the person was photographed from the back (row\n6, column 6). Results on FLIC (see Fig. 9) are usually better\nwith occasional visible mistakes on lower arms.\n5. Conclusion\nWe present, to our knowledge, the \ufb01rst application of\nDeep Neural Networks (DNNs) to human pose estimation.\nOur formulation of the problem as DNN-based regression to"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f6-6db4-11ee-8a3c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-31", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "Deep Neural Networks (DNNs) to human pose estimation.\nOur formulation of the problem as DNN-based regression to\njoint coordinates and the presented cascade of such regressors has the advantage of capturing context and reasoning\nabout pose in a holistic manner. As a result, we are able to\nachieve state-of-art or better results on several challenging\nacademic datasets.\nFurther, we show that using a generic convolutional neural network, which was originally designed for classi\ufb01cation tasks, can be applied to the different task of localization. In future, we plan to investigate novel architectures\nwhich could be potentially better tailored towards localization problems in general, and in pose estimation in particular.\nAcknowledgements I would like to thank Luca Bertelli,\nBen Sapp and Tianli Yu for assistance with data and fruitful\ndiscussions.\nReferences\n[1] M. Andriluka, S. Roth, and B. Schiele. Pictorial structures\nrevisited: People detection and articulated pose estimation.\nInCVPR , 2009.\n[2] M. Dantone, J. Gall, C. Leistner, and L. Van Gool. Human\npose estimation using body parts dependent joint regressors.\nInCVPR , 2013.\n[3] J. Duchi, E. Hazan, and Y . Singer. Adaptive subgradient"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f7-6db4-11ee-9d04-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-32", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "pose estimation using body parts dependent joint regressors.\nInCVPR , 2013.\n[3] J. Duchi, E. Hazan, and Y . Singer. Adaptive subgradient\nmethods for online learning and stochastic optimization. In\nCOLT . ACL, 2010.\n[4] M. Eichner and V . Ferrari. Better appearance models for\npictorial structures. 2009.\n[5] M. Eichner, M. Marin-Jimenez, A. Zisserman, and V . Ferrari.\nArticulated human pose estimation and search in (almost)\nunconstrained still images. ETH Zurich, D-ITET, BIWI,\nTechnical Report No , 272, 2010.\n[6] P. F. Felzenszwalb and D. P. Huttenlocher. Pictorial structures for object recognition. International Journal of Computer Vision , 61(1):55\u201379, 2005.\n[7] V . Ferrari, M. Marin-Jimenez, and A. Zisserman. Progressive\nsearch space reduction for human pose estimation. In CVPR ,\n2008.\n[8] M. A. Fischler and R. A. Elschlager. The representation and\nmatching of pictorial structures. Computers, IEEE Transactions on , 100(1):67\u201392, 1973."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f8-6db4-11ee-80ae-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-33", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "matching of pictorial structures. Computers, IEEE Transactions on , 100(1):67\u201392, 1973.\nFigure 8. Visualization of pose results on images from LSP. Each pose is represented as a stick \ufb01gure, inferred from predicted joints.\nDifferent limbs in the same image are colored differently, same limb across different images has the same color.\nFigure 9. Visualization of pose results on images from FLIC. Meaning of stick \ufb01gures is the same as in Fig. 8 above.\n[9] R. Girshick, J. Donahue, T. Darrell, and J. Malik. Rich feature hierarchies for accurate object detection and semantic\nsegmentation. In CVPR , 2014.\n[10] G. Gkioxari, P. Arbel \u00b4aez, L. Bourdev, and J. Malik. Articulated pose estimation using discriminative armlet classi\ufb01ers.\nInCVPR , 2013.\n[11] C. Ionescu, F. Li, and C. Sminchisescu. Latent structured\nmodels for human pose estimation. In ICCV , 2011.\n[12] S. Johnson and M. Everingham. Clustered pose and nonlinear appearance models for human pose estimation. In BMVC ,\n2010.\n[13] S. Johnson and M. Everingham. Learning effective human"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1f9-6db4-11ee-8faf-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-34", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "2010.\n[13] S. Johnson and M. Everingham. Learning effective human\npose estimation from inaccurate annotation. In CVPR , 2011.\n[14] A. Krizhevsky, I. Sutskever, and G. Hinton. Imagenet classi\ufb01cation with deep convolutional neural networks. In NIPS ,\n2012.\n[15] G. Mori and J. Malik. Estimating human body con\ufb01gurations\nusing shape context matching. In ECCV , 2002.[16] R. Nevatia and T. O. Binford. Description and recognition of\ncurved objects. Arti\ufb01cial Intelligence , 8(1):77\u201398, 1977.\n[17] M. Osadchy, Y . LeCun, and M. L. Miller. Synergistic face\ndetection and pose estimation with energy-based models.\nThe Journal of Machine Learning Research , 8:1197\u20131215,\n2007.\n[18] L. Pishchulin, M. Andriluka, P. Gehler, and B. Schiele. Poselet conditioned pictorial structures. In CVPR , 2013.\n[19] D. Ramanan. Learning to parse images of articulated bodies.\nInNIPS , 2006."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1fa-6db4-11ee-b65b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-35", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "[19] D. Ramanan. Learning to parse images of articulated bodies.\nInNIPS , 2006.\n[20] B. Sapp and B. Taskar. Modec: Multimodal decomposable\nmodels for human pose estimation. In CVPR , 2013.\n[21] G. Shakhnarovich, P. Viola, and T. Darrell. Fast pose estimation with parameter-sensitive hashing. In CVPR , 2003.\n[22] Y . Sun, X. Wang, and X. Tang. Deep convolutional network cascade for facial point detection. In Computer Vision\nand Pattern Recognition (CVPR), 2013 IEEE Conference on ,\npages 3476\u20133483. IEEE, 2013.\n[23] C. Szegedy, A. Toshev, and D. Erhan. Object detection via\ndeep neural networks. In NIPS 26 , 2013.\n[24] G. W. Taylor, R. Fergus, G. Williams, I. Spiro, and C. Bregler. Pose-sensitive embedding by nonlinear nca regression.\nInNIPS , 2010.\n[25] Y . Tian, C. L. Zitnick, and S. G. Narasimhan. Exploring the\nspatial hierarchy of mixture models for human pose estimation. In ECCV , 2012."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1fb-6db4-11ee-a452-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1312.4659-36", "title": "DeepPose: Human Pose Estimation via Deep Neural Networks", "source": "http://arxiv.org/pdf/1312.4659", "start_index": 0, "chroma:document": "spatial hierarchy of mixture models for human pose estimation. In ECCV , 2012.\n[26] F. Wang and Y . Li. Beyond physical connections: Tree models in human pose estimation. In CVPR , 2013.\n[27] Y . Yang and D. Ramanan. Articulated pose estimation with\n\ufb02exible mixtures-of-parts. In CVPR , 2011."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1fc-6db4-11ee-8914-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-0", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "1\nReturn of the Devil in the Details:\nDelving Deep into Convolutional Nets\nKen Chat\ufb01eld, Karen Simonyan, Andrea Vedaldi, and Andrew Zisserman\nVisual Geometry Group, Department of Engineering Science, University of Oxford\nfken,karen,vedaldi,az g@robots.ox.ac.uk\nAbstract \u2014The latest generation of Convolutional Neural Networks (CNN) have achieved impressive results in challenging benchmarks on image recognition and object detection, signi\ufb01cantly raising the interest of the community in\nthese methods. Nevertheless, it is still unclear how different CNN methods compare with each other and with previous\nstate-of-the-art shallow representations such as the Bag-of-Visual-Words and the Improved Fisher Vector. This paper\nconducts a rigorous evaluation of these new techniques, exploring different deep architectures and comparing them on\na common ground, identifying and disclosing important implementation details. We identify several useful properties\nof CNN-based representations, including the fact that the dimensionality of the CNN output layer can be reduced\nsigni\ufb01cantly without having an adverse effect on performance. We also identify aspects of deep and shallow methods\nthat can be successfully shared. In particular, we show that the data augmentation techniques commonly applied to\nCNN-based methods can also be applied to shallow methods, and result in an analogous performance boost. Source"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1fd-6db4-11ee-b24c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-1", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "CNN-based methods can also be applied to shallow methods, and result in an analogous performance boost. Source\ncode and models to reproduce the experiments in the paper is made publicly available.\nF\n1 I NTRODUCTION\nPERHAPS the single most important design\nchoice in current state-of-the-art image classi\ufb01cation and object recognition systems is the choice\nof visual features, or image representation. In fact,\nmost of the quantitative improvements to image\nunderstanding obtained in the past dozen years can\nbe ascribed to the introduction of improved representations, from the Bag-of-Visual-Words (BoVW) [1],\n[2] to the (Improved) Fisher Vector (IFV) [3]. A\ncommon characteristic of these methods is that\nthey are largely handcrafted . They are also relatively\nsimple, comprising dense sampling of local image\npatches, describing them by means of visual descriptors such as SIFT, encoding them into a highdimensional representation, and then pooling over\nthe image. Recently, these handcrafted approaches\nhave been substantially outperformed by the introduction of the latest generation of Convolutional\nNeural Networks (CNNs) [4] to the computer vision\n\ufb01eld. These networks have a substantially more\nsophisticated structure than standard representations, comprising several layers of non-linear feature extractors, and are therefore said to be deep"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1fe-6db4-11ee-825e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-2", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "\ufb01eld. These networks have a substantially more\nsophisticated structure than standard representations, comprising several layers of non-linear feature extractors, and are therefore said to be deep\n(in contrast, classical representation will be referred\nto as shallow ). Furthermore, while their structureis handcrafted, they contain a very large number\nof parameters learnt from data. When applied to\nstandard image classi\ufb01cation and object detection\nbenchmark datasets such as ImageNet ILSVRC [5]\nand PASCAL VOC [6] such networks have demonstrated excellent performance [7], [8], [9], [10], [11],\nsigni\ufb01cantly better than standard image encodings [12].\nDespite these impressive results, it remains unclear how different deep architectures compare to\neach other and to shallow computer vision methods such as IFV . Most papers did not test these\nrepresentations extensively on a common ground,\nso a systematic evaluation of the effect of different design and implementation choices remains\nlargely missing. As noted in our previous work [12],\nwhich compared the performance of various shallow visual encodings, the performance of computer\nvision systems depends signi\ufb01cantly on implementation\ndetails . For example, state-of-the-art methods such\nas [13] not only involve the use of a CNN, but\nalso include other improvements such as the use"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f1ff-6db4-11ee-b8ec-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-3", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "details . For example, state-of-the-art methods such\nas [13] not only involve the use of a CNN, but\nalso include other improvements such as the use\nof very large scale datasets, GPU computation, and\ndata augmentation (also known as data jittering or\nvirtual sampling). These improvements could also\ntransfer to shallow representations such as the IFV ,\npotentially explaining a part of the performancearXiv:1405.3531v4  [cs.CV]  5 Nov 2014\n2\ngap [14].\nIn this study we analyse and empirically clarify\nthese issues, conducting a large set of rigorous\nexperiments (Sect. 4), in many ways picking up\nthe story where it last ended in [12] with the\ncomparison of shallow encoders. We focus on methods to construct image representations ,i.e. encoding functions \u001emapping an image Ito a vector\n\u001e(I)2Rdsuitable for analysis with a linear classi\ufb01er, such as an SVM. We consider three scenarios\n(Sect. 2, Sect. 3): shallow image representations,\ndeep representations pre-trained on outside data,\nand deep representation pre-trained and then \ufb01netuned on the target dataset. As part of our tests,\nwe explore generally-applicable best practices that"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f200-6db4-11ee-bec5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-4", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "and deep representation pre-trained and then \ufb01netuned on the target dataset. As part of our tests,\nwe explore generally-applicable best practices that\nare nevertheless more often found in combination with CNNs [13] or, alternatively, with shallow\nencoders [12], porting them with mutual bene\ufb01t.\nThese are (Sect. 2): the use of colour information ,\nfeature normalisation , and, most importantly, the\nuse of substantial data augmentation . We also determine scenario-speci\ufb01c best-practices , improving\nthe ones in [12], [15] and others, including dimensionality reduction for deep features. Finally, we\nachieve performance competitive with the state\nof the art [16], [17] on PASCAL VOC classi\ufb01cation\nusing less additional training data and signi\ufb01cantly\nsimpler techniques. As in [12], the source code and\nmodels to reproduce all experiments in this paper\nis available on the project website1.\n2 S CENARIOS\nThis section introduces the three types of image\nrepresentation \u001e(I)considered in this paper, describing them within the context of three different\nscenarios. Having outlined details speci\ufb01c to each,\ngeneral methodologies which apply to all three\nscenarios are reviewed, such as data augmentation"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f201-6db4-11ee-9d78-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-5", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "scenarios. Having outlined details speci\ufb01c to each,\ngeneral methodologies which apply to all three\nscenarios are reviewed, such as data augmentation\nand feature normalisation, together with the linear\nclassi\ufb01er (trained with a standard hinge loss). We\nalso specify here the benchmark datasets used in\nthe evaluation.\n2.1 Scenario 1: Shallow representation (IFV)\nOur reference shallow image representation is the\nIFV [3]. Our choice is motivated by the fact that\nIFV usually outperforms related encoding methods\nsuch as BoVW, LLC [12], and VLAD [18]. Given an\nimageI, the IFV\u001eFV(I)is obtained by extracting\n1. http://www.robots.ox.ac.uk/\u0018vgg/research/deep eval/a dense collection of patches and corresponding\nlocal descriptors xi2RD(e.g. SIFT) from the image\nat multiple scales. Each descriptor xiis then softquantized using a Gaussian Mixture Model with\nKcomponents. First and second order differences\nbetween each descriptor xiand its Gaussian cluster\nmean\u0016kare accumulated in corresponding blocks\nuk,vkin the vector \u001eFV(I)2R2KD, appropriately\nweighed by the Gaussian soft-assignments and"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f202-6db4-11ee-903a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-6", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "uk,vkin the vector \u001eFV(I)2R2KD, appropriately\nweighed by the Gaussian soft-assignments and\ncovariance, leading to a 2KD-dimensional image\nrepresentation \u001eFV(I) = [u>\n1;v>\n1;:::u>\nK;v>\nK]>:The\nimproved version of the Fisher vector involves postprocessing \u001eFVby computing the signed squareroot of its scalar components and normalising the\nresult to a unit `2norm. The details of this construction can be found in [3]; here we follow the\nnotation of [12].\n2.2 Scenario 2: Deep representation (CNN) with\npre-training\nOur deep representations are inspired by the success of the CNN of Krizhevsky et al. [13]. As shown\nin [7], [19], the vector of activities \u001eCNN(I)of the\npenultimate layer of a deep CNN, learnt on a large\ndataset such as ImageNet [5], can be used as a powerful image descriptor applicable to other datasets.\nNumerous CNN architectures that improve the\nprevious state of the art obtained using shallow\nrepresentations have been proposed, but choosing"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f203-6db4-11ee-a704-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-7", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "Numerous CNN architectures that improve the\nprevious state of the art obtained using shallow\nrepresentations have been proposed, but choosing\nthe best one remains an open question. Many are inspired by [13]: DeCAF [7], [11], Caffe [20], Oquab et\nal. [8]. Others use larger networks with a smaller\nstride of the \ufb01rst convolutional layer: Zeiler and\nFergus [19] and OverFeat [10], [9]. Other differences\ninclude the CNN pre-training protocols. Here we\nadopt a single learning framework and experiment\nwith architectures of different complexity exploring\ntheir performance-speed trade-off.\n2.3 Scenario 3: Deep representation (CNN) with\npre-training and \ufb01ne-tuning\nIn Scenario 2 features are trained on one (large)\ndataset and applied to another (usually smaller).\nHowever, it was demonstrated [11] that \ufb01ne-tuning\na pre-trained CNN on the target data can signi\ufb01cantly improve the performance. We consider\nthis scenario separately from that of Scenario 2, as\nthe image features become dataset-speci\ufb01c after the\n\ufb01ne-tuning.\n3\n2.4 Commonalities\nWe now turn to what is in common across the\nscenarios."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f204-6db4-11ee-b3cf-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-8", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "\ufb01ne-tuning.\n3\n2.4 Commonalities\nWe now turn to what is in common across the\nscenarios.\n2.4.1 Data augmentation\nData augmentation is a method applicable to shallow and deep representations, but that has been\nso far mostly applied to the latter [13], [19]. By\naugmentation we mean perturbing an image Iby\ntransformations that leave the underlying class unchanged ( e.g. cropping and \ufb02ipping) in order to\ngenerate additional examples of the class. Augmentation can be applied at training time, at test time,\nor both. The augmented samples can either be taken\nas-is or combined to form a single feature, e.g. using\nsum/max-pooling or stacking.\n2.4.2 Linear predictors\nAll the representations \u001e(I)in the three scenarios\nare used to construct linear predictorshw;\u001e(I)ifor\neach class to be recognized. These predictors are\nlearnt using Support Vector Machines (SVM) by\n\ufb01tting wto the available training data by minimizing an objective function balancing a quadratic\nregularizer and the hinge-loss. The parameter C\nin the SVM, trading-off regularizer and loss, is\ndetermined using an held-off validation subset of"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f205-6db4-11ee-9f1c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-9", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "regularizer and the hinge-loss. The parameter C\nin the SVM, trading-off regularizer and loss, is\ndetermined using an held-off validation subset of\nthe data. Here we use the same learning framework\nwith all representations. It is common experience\nthat linear classi\ufb01ers are particularly sensitive to\nthenormalisation of the data and that, in particular,\nSVMs tend to bene\ufb01t from `2normalisation [3] (an\ninterpretation is that after normalisation the inner\nproduct corresponds to the cosine similarly).\n2.5 Benchmark data\nAs reference benchmark we use the PASCAL\nVOC [6] data as already done in [12]. The VOC2007 edition contains about 10,000 images split into\ntrain, validation, and test sets, and labelled with\ntwenty object classes. A one-vs-rest SVM classi\ufb01er\nfor each class is learnt and evaluated independently\nand the performance is measured as mean Average\nPrecision (mAP) across all classes. The VOC-2012\nedition contains roughly twice as many images\nand does not include test labels; instead, evaluation\nuses the of\ufb01cial PASCAL Evaluation Server. To train\ndeep representations we use the ILSVRC-2012 challenge dataset. This contains 1,000 object categories"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f206-6db4-11ee-b4e0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-10", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "uses the of\ufb01cial PASCAL Evaluation Server. To train\ndeep representations we use the ILSVRC-2012 challenge dataset. This contains 1,000 object categories\nfrom ImageNet [5] with roughly 1.2M training images, 50,000 validation images, and 100,000 testimages. Performance is evaluated using the top-5\nclassi\ufb01cation error. Finally, we also evaluate over\ntheCaltech-101 and Caltech-256 image classi\ufb01cation benchmarks [21], [22]. For Caltech-101, we\nfollowed the protocol of [12], and considered three\nrandom splits into training and testing data, each\nof which comprises 30 training and up to 30 testing\nimages per class. For Caltech-256, two random\nsplits were generated, each of which contains 60\ntraining images per class, and the rest are used for\ntesting. On both Caltech datasets, performance is\nmeasured using mean class accuracy.\n3 D ETAILS\nThis section gives the implementation details of the\nmethods introduced in Sect. 2.\n3.1 Improved Fisher Vector details\nOur IFV representation uses a slightly improved\nsetting compared to the best result of [12].\nComputation starts by upscaling the image Iby a\nfactor of 2 [23], followed by SIFT features extraction"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f207-6db4-11ee-9677-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-11", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "setting compared to the best result of [12].\nComputation starts by upscaling the image Iby a\nfactor of 2 [23], followed by SIFT features extraction\nwith a stride of 3 pixels at 7 different scales withp\n2\nscale increments. These features are square-rooted\nas suggested by [24], and decorrelated and reduced\nin dimension from 128Dto80Dusing PCA. A\nGMM with K= 256 components is learnt from features sampled from the training images. Hence the\nFisher Vector \u001eFV(I)has dimension 2KD= 40;960.\nBefore use in classi\ufb01cation, the vector is signedsquare-rooted and l2-normalised (square rooting\ncorrespond to the Hellinger\u2019s kernel map [25]). As\nin [12], square-rooting is applied twice, once to the\nraw encodings, and once again after sum pooling\nand normalisation. In order to capture weak geometrical information, the IFV representation is used\nin a spatial pyramid [26]. As in [12], the image is\ndivided into 1\u00021,3\u00021, and 2\u00022spatial subdivisions\nand corresponding IFVs are computed and stacked\nwith an overall dimension of 8\u00022KD= 327;680\nelements."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f208-6db4-11ee-9da7-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-12", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "and corresponding IFVs are computed and stacked\nwith an overall dimension of 8\u00022KD= 327;680\nelements.\nIn addition to this standard formulation, we experiment with a few modi\ufb01cations. The \ufb01rst one\nis the use of intra-normalisation of the descriptor\nblocks, an idea recently proposed for the VLAD\ndescriptor [27]. In this case, the `2normalisation\nis applied to the individual sub-blocks (uk;vk)of\nthe vector \u001eFV(I), which helps to alleviate the\nlocal feature burstiness [28]. In the case of the\nimproved intra-normalised features, it was found\n4\nthat applying the square-rooting only once to the\n\ufb01nal encoding produced the best results.\nThe second modi\ufb01cation is the use of spatiallyextended local descriptors [23] instead of a spatial\npyramid. Here descriptors xiare appended with\ntheir image location (xi;yi)before quantization\nwith the GMM. Formally, xiis extended, after PCA\nprojection, with its normalised spatial coordinates:\n[x>\ni;xi=W\u00000:5;yi=H\u00000:5]>, whereW\u0002Hare the\ndimensions of the image. Since the GMM quantizes"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f209-6db4-11ee-9412-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-13", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "i;xi=W\u00000:5;yi=H\u00000:5]>, whereW\u0002Hare the\ndimensions of the image. Since the GMM quantizes\nboth appearance and location, this allows for spatial\ninformation to be captured directly by the softquantization process. This method is signi\ufb01cantly\nmore memory-ef\ufb01cient than using a spatial pyramid. Speci\ufb01cally, the PCA-reduced SIFT features\nare spatially augmented by appending (x;y)yieldingD= 82 dimensional descriptors pooled in a\n2KD= 41;984dimensional IFV .\nThe third modi\ufb01cation is the use of colour features in addition to SIFT descriptors. While colour\ninformation is used in CNNs [13] and by the original FV paper [3], it was not explored in our previous comparison [12]. We do so here by adopting\nthe same Local Colour Statistics (LCS) features as\nused by [3]. LCS is computed by dividing an input\npatch into a 4\u00024spatial grid (akin to SIFT), and\ncomputing the mean and variance of each of the\nLab colour channels for each cell of the grid. The\nLCS dimensionality is thus 4\u00024\u00022\u00023 = 96 . This\nis then encoded in a similar manner to SIFT."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f20a-6db4-11ee-800b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-14", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "Lab colour channels for each cell of the grid. The\nLCS dimensionality is thus 4\u00024\u00022\u00023 = 96 . This\nis then encoded in a similar manner to SIFT.\n3.2 Convolutional neural networks details\nThe CNN-based features are based on three CNN\narchitectures representative of the state of the art\n(shown in Table 1) each exploring a different accuracy/speed trade-off. To ensure a fair comparison\nbetween them, these networks are trained using the\nsame training protocol and the same implementation, which we developed based on the opensource Caffe framework [20]. `2-normalising the\nCNN features \u001eCNN(I)before use in the SVM was\nfound to be important for performance.\nOur Fast (CNN-F) architecture is similar to the\none used by Krizhevsky et al . [13]. It comprises\n8 learnable layers, 5 of which are convolutional,\nand the last 3 are fully-connected. The input image\nsize is 224\u0002224. Fast processing is ensured by the\n4 pixel stride in the \ufb01rst convolutional layer. The\nmain differences between our architecture and that\nof [13] are the reduced number of convolutional\nlayers and the dense connectivity between convolu-tional layers ([13] used sparse connections to enable\ntraining on two GPUs)."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f20b-6db4-11ee-ac0e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-15", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "of [13] are the reduced number of convolutional\nlayers and the dense connectivity between convolu-tional layers ([13] used sparse connections to enable\ntraining on two GPUs).\nOur Medium (CNN-M) architecture is similar\nto the one used by Zeiler and Fergus [19]. It is\ncharacterised by the decreased stride and smaller\nreceptive \ufb01eld of the \ufb01rst convolutional layer, which\nwas shown to be bene\ufb01cial on the ILSVRC dataset.\nAt the same time, conv2 uses larger stride (2 instead\nof 1) to keep the computation time reasonable. The\nmain difference between our net and that of [19] is\nwe use less \ufb01lters in the conv4 layer (512 vs. 1024).\nOur Slow (CNN-S) architecture is related to the\n\u2018accurate\u2019 network from the OverFeat package [10].\nIt also uses 7\u00027\ufb01lters with stride 2in conv1.\nUnlike CNN-M and [19], the stride in conv2 is\nsmaller (1 pixel), but the max-pooling window in\nconv1 and conv5 is larger ( 3\u00023) to compensate for\nthe increased spatial resolution. Compared to [10],\nwe use 5 convolutional layers as in the previous"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f20c-6db4-11ee-8fc4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-16", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "conv1 and conv5 is larger ( 3\u00023) to compensate for\nthe increased spatial resolution. Compared to [10],\nwe use 5 convolutional layers as in the previous\narchitectures ([10] used 6), and less \ufb01lters in conv5\n(512 instead of 1024); we also incorporate an LRN\nlayer after conv1 ([10] did not use contrast normalisation).\n3.2.1 CNN training\nIn general, our CNN training procedure follows that of [13], learning on ILSVRC-2012 using gradient descent with momentum. The hyperparameters are the same as used by [13]: momentum0:9; weight decay 5\u000110\u00004; initial learning rate\n10\u00002, which is decreased by a factor of 10, when\nthe validation error stop decreasing. The layers are\ninitialised from a Gaussian distribution with a zero\nmean and variance equal to 10\u00002. We also employ\nsimilar data augmentation in the form of random\ncrops, horizontal \ufb02ips, and RGB colour jittering.\nTest time crop sampling is discussed in Sect. 3.3;\nat training time, 224\u0002224 crops are sampled randomly, rather than deterministically. Thus, the only\nnotable difference to [13] is that the crops are taken\nfrom the whole training image P\u0002256;P\u0015256,"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f20d-6db4-11ee-822b-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-17", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "notable difference to [13] is that the crops are taken\nfrom the whole training image P\u0002256;P\u0015256,\nrather than its 256\u0002256 centre. Training was performed on a single NVIDIA GTX Titan GPU and\nthe training time varied from 5 days for CNN-F to\n3 weeks for CNN-S.\n3.2.2 CNN \ufb01ne-tuning on the target dataset\nIn our experiments, we \ufb01ne-tuned CNN-S using\nVOC-2007, VOC-2012, or Caltech-101 as the target\ndata. Fine-tuning was carried out using the same\nframework (and the same data augmentation), as\nwe used for CNN training on ILSVRC. The last\n5\nArch. conv1 conv2 conv3 conv4 conv5 full6 full7 full8\nCNN-F64x11x11 256x5x5 256x3x3 256x3x3 256x3x3 4096 4096 1000\nst. 4, pad 0 st. 1, pad 2 st. 1, pad 1 st. 1, pad 1 st. 1, pad 1 drop- drop- softLRN, x2 pool LRN, x2 pool - - x2 pool out out max"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f20e-6db4-11ee-b3d6-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-18", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "CNN-M96x7x7 256x5x5 512x3x3 512x3x3 512x3x3 4096 4096 1000\nst. 2, pad 0 st. 2, pad 1 st. 1, pad 1 st. 1, pad 1 st. 1, pad 1 drop- drop- softLRN, x2 pool LRN, x2 pool - - x2 pool out out max\nCNN-S96x7x7 256x5x5 512x3x3 512x3x3 512x3x3 4096 4096 1000\nst. 2, pad 0 st. 1, pad 1 st. 1, pad 1 st. 1, pad 1 st. 1, pad 1 drop- drop- softLRN, x3 pool x2 pool - - x3 pool out out max\nTABLE 1\nCNN architectures. Each architecture contains 5 convolutional layers (conv 1\u20135) and three\nfully-connected layers (full 1\u20133). The details of each of the convolutional layers are given in three sub-rows:\nthe \ufb01rst speci\ufb01es the number of convolution \ufb01lters and their receptive \ufb01eld size as \u201cnum x size x size\u201d; the\nsecond indicates the convolution stride (\u201cst.\u201d) and spatial padding (\u201cpad\u201d); the third indicates if Local"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f20f-6db4-11ee-9eff-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-19", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "second indicates the convolution stride (\u201cst.\u201d) and spatial padding (\u201cpad\u201d); the third indicates if Local\nResponse Normalisation (LRN) [13] is applied, and the max-pooling downsampling factor. For full 1\u20133, we\nspecify their dimensionality, which is the same for all three architectures. Full6 and full7 are regularised\nusing dropout [13], while the last layer acts as a multi-way soft-max classi\ufb01er. The activation function for all\nweight layers (except for full8) is the REcti\ufb01cation Linear Unit (RELU) [13].\nfully-connected layer (conv8) has output dimensionality equal to the number of classes, which\ndiffers between datasets, so we initialised it from\na Gaussian distribution (as used for CNN training\nabove). Now we turn to dataset-speci\ufb01c \ufb01ne-tuning\ndetails.\nVOC-2007 and VOC-2012. Considering that PASCAL VOC is a multi-label dataset ( i.e. a single\nimage might have multiple labels), we replaced the\nsoftmax regression loss with a more appropriate\nloss function, for which we considered two options:\none-vs-rest classi\ufb01cation hinge loss (the same loss"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f210-6db4-11ee-a300-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-20", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "softmax regression loss with a more appropriate\nloss function, for which we considered two options:\none-vs-rest classi\ufb01cation hinge loss (the same loss\nas used in the SVM experiments) and ranking hinge\nloss. Both losses de\ufb01ne constraints on the scores of\npositive (Ipos) and negative ( Ineg) images for each\nclass:wc\u001e(Ipos)>1\u0000\u0018;wc\u001e(Ineg)<\u00001 +\u0018for the\nclassi\ufb01cation loss, wc\u001e(Ipos)> w c\u001e(Ineg) + 1\u0000\u0018\nfor the ranking loss ( wcis thec-th row of the\nlast fully-connected layer, which can be seen as a\nlinear classi\ufb01er on deep features \u001e(I);\u0018is a slack\nvariable). Our \ufb01ne-tuned networks are denoted as\n\u201cCNN S TUNE-CLS\u201d (for the classi\ufb01cation loss)\nand \u201cCNN S TUNE-RNK\u201d (for the ranking loss).\nIn the case of both VOC datasets, the training\nand validation subsets were combined to form a\nsingle training set. Given the smaller size of the\ntraining data when compared to ILSVRC-2012, we"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f211-6db4-11ee-9265-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-21", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "and validation subsets were combined to form a\nsingle training set. Given the smaller size of the\ntraining data when compared to ILSVRC-2012, we\ncontrolled for over-\ufb01tting by using lower initial\nlearning rates for the \ufb01ne-tuned hidden layers.The learning rate schedule for the last layer /\nhidden layers was: 10\u00002=10\u00004!10\u00003=10\u00004!\n10\u00004=10\u00004!10\u00005=10\u00005.\nCaltech-101 dataset contains a single class label\nper image, so \ufb01ne-tuning was performed using the\nsoftmax regression loss. Other settings (including\nthe learning rate schedule) were the same as used\nfor the VOC \ufb01ne-tuning experiments.\n3.2.3 Low-dimensional CNN feature training\nOur baseline networks (Table 1) have the same\ndimensionality of the last hidden layer (full7): 4096.\nThis design choice is in accordance with the stateof-the-art architectures [13], [19], [10], and leads to\na 4096-D dimensional image representation, which\nis already rather compact compared to IFV . We\nfurther trained three modi\ufb01cations of the CNN-M\nnetwork, with lower dimensional full7 layers of:\n2048, 1024, and 128 dimensions respectively. The"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f212-6db4-11ee-ab4c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-22", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "further trained three modi\ufb01cations of the CNN-M\nnetwork, with lower dimensional full7 layers of:\n2048, 1024, and 128 dimensions respectively. The\nnetworks were learnt on ILSVRC-2012. To speedup training, all layers aside from full7/full8 were\nset to those of the CNN-M net and a lower initial\nlearning rate of 10\u00003was used. The initial learning\nrate of full7/full8 was set to 10\u00002.\n3.3 Data augmentation details\nWe explore three data augmentation strategies. The\n\ufb01rst strategy is to use no augmentation . In contrast to IFV , however, CNNs require images to\n6\nbe transformed to a \ufb01xed size ( 224\u0002224) even\nwhen no augmentation is used. Hence the image is\ndownsized so that the smallest dimension is equal\nto224 pixels and a 224\u0002224 crop is extracted\nfrom the centre.2The second strategy is to use\n\ufb02ip augmentation , mirroring images about the yaxis producing two samples from each image. The\nthird strategy, termed C+F augmentation , combines\ncropping and \ufb02ipping. For CNN-based representations, the image is downsized so that the smallest\ndimension is equal to 256 pixels. Then 224\u0002224"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f213-6db4-11ee-9158-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-23", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "cropping and \ufb02ipping. For CNN-based representations, the image is downsized so that the smallest\ndimension is equal to 256 pixels. Then 224\u0002224\ncrops are extracted from the four corners and the\ncentre of the image. Note that the crops are sampled\nfrom the whole image, rather than its 256\u0002256\ncentre, as done by [13]. These crops are then \ufb02ipped\nabout they-axis, producing 10perturbed samples\nper input image. In the case of the IFV encoding, the\nsame crops are extracted, but at the original image\nresolution.\n4 A NALYSIS\nThis section describes the experimental results,\ncomparing different features and data augmentation schemes. The results are given in Table 2 for\nVOC-2007 and analysed next, starting from generally applicable methods such as augmentation and\nthen discussing the speci\ufb01cs of each scenario. We\nthen move onto other datasets and the state of the\nart in Sect. 4.7.\n4.1 Data augmentation\nWe experiment with no data augmentation (denoted Image Aug=\u2013 in Tab. 2), \ufb02ip augmentation ( Image Aug=F ), and C+F augmentation ( Image\nAug=C ). Augmented images are used as standalone samples ( f), or by fusing the corresponding"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f214-6db4-11ee-a4b6-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-24", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "Aug=C ). Augmented images are used as standalone samples ( f), or by fusing the corresponding\ndescriptors using sum ( s) or max ( m) pooling or\nstacking ( t). So for example Image Aug=(C) f s in\nrow [f] of Tab. 2 means that C+F augmentation is\nused to generate additional samples in training ( f),\nand is combined with sum-pooling in testing ( s).\nAugmentation consistently improves performance by\u00183%for both IFV ( e.g. [d] vs. [f]) and\nCNN ( e.g. [o] vs. [p]). Using additional samples for\ntraining and sum-pooling for testing works best\n([p]) followed by sum-pooling [r], max pooling [q],\nand stacking [s]. In terms of the choice of transformations, \ufb02ipping improves only marginally ([o] vs.\n2. Extracting a 224\u0002224 centre crop from a 256\u0002256\nimage [13] resulted in worse performance.[u]), but using the more expensive C+F sampling\nimproves, as seen, by about 2\u00183%([o]vs. [p]). We\nexperimented with sampling more transformations,\ntaking a higher density of crops from the centre of\nthe image, but observed no bene\ufb01t.\n4.2 Colour"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f215-6db4-11ee-b9e0-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-25", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "experimented with sampling more transformations,\ntaking a higher density of crops from the centre of\nthe image, but observed no bene\ufb01t.\n4.2 Colour\nColour information can be added and subtracted in\nCNN and IFV . In IFV replacing SIFT with the colour\ndescriptors of [3] (denoted COL inMethod ) yields\nsigni\ufb01cantly worse performance ([j] vs. [h]). However, when SIFT and colour descriptors are combined by stacking the corresponding IFVs ( COL+ )\nthere is a small but signi\ufb01cant improvement of\naround\u00181%in the non-augmented case ( e.g. [h]\nvs. [k]) but little impact in the augmented case\n(e.g. [i] vs. [l]). For CNNs, retraining the network\nafter converting all the input images to grayscale\n(denoted GSinMethods ) has a more signi\ufb01cant\nimpact, resulting in a performance drop of \u00183%\n([w] vs. [p], [v] vs. [o]).\n4.3 Scenario 1: Shallow representation (IFV)\nThe baseline IFV encoding using a spatial pyramid\n[a] performs slightly better than the results [I] taken"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d19f216-6db4-11ee-8c03-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-26", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "4.3 Scenario 1: Shallow representation (IFV)\nThe baseline IFV encoding using a spatial pyramid\n[a] performs slightly better than the results [I] taken\nfrom Chat\ufb01eld et al. [12], primarily due to a larger\nnumber of spatial scales being used during SIFT\nfeature extraction, and the resultant SIFT features\nbeing square-rooted. Intra-normalisation , denoted as\nINin the Method column of the table, improves the\nperformance by \u00181%(e.g. [c] vs. [d]). More interestingly, switching from spatial pooling (denoted\nspm in the SPool column) to feature spatial augmentation ( SPool=(x,y) ) has either little effect on the\nperformance or results in a marginal increase ([a] vs.\n[c], [b] vs. [d]), whilst resulting in a representation\nwhich is over 10 \u0002smaller. We also experimented\nwith augmenting with scale in addition to position\nas in [23] but observed no improvement. Finally,\nwe investigate pushing the parameters of the representation setting K= 512 (rows [h]-[l]). Increasing\nthe number of GMM centres in the model from\nK= 256 to512 results in a further performance\nincrease (e.g. [h] vs. [d]), but at the expense of"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a187c-6db4-11ee-870f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-27", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "K= 256 to512 results in a further performance\nincrease (e.g. [h] vs. [d]), but at the expense of\nhigher-dimensional codes (125K dimensional).\n4.4 Scenario 2: Deep representation (CNN) with\npre-training\nCNN-based methods consistently outperform the\nshallow encodings, even after the improvements\n7\nMethod SPool Image Aug. Dim mAP\n(I) FK BL spm \u2013 327K 61.69 79.0 67.4 51.9 70.9 30.8 72.2\n(II) DECAF \u2013 (C) t t 327K 73.41 87.4 79.3 84.1 78.4 42.3 73.7\n(a) FK spm \u2013 327K 63.66 83.4 68.8 59.6 74.1 35.7 71.2\n(b) FK IN spm \u2013 327K 64.18 82.1 69.7 59.7 75.2 35.7 71.3\n(c) FK (x,y) \u2013 42K 63.51 83.2 69.4 60.6 73.9 36.3 68.6\n(d) FK IN (x,y) \u2013 42K 64.36 83.1 70.4 62.4 75.2 37.1 69.1"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a187d-6db4-11ee-8919-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-28", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(d) FK IN (x,y) \u2013 42K 64.36 83.1 70.4 62.4 75.2 37.1 69.1\n(e) FK IN (x,y) (F) f - 42K 64.35 83.1 70.5 62.3 75.4 37.1 69.1\n(f) FK IN (x,y) (C) f s 42K 67.17 85.5 71.6 64.6 77.2 39.0 70.8\n(g) FK IN (x,y) (C) s s 42K 66.68 84.9 70.1 64.7 76.3 39.2 69.8\n(h) FK IN 512 (x,y) \u2013 84K 65.36 84.1 70.4 65.0 76.7 37.2 71.3\n(i) FK IN 512 (x,y) (C) f s 84K 68.02 85.9 71.8 67.1 77.1 38.8 72.3\n(j) FK IN COL 512 \u2013 \u2013 82K 52.18 69.5 52.1 47.5 64.0 24.6 49.8"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a187e-6db4-11ee-bf44-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-29", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(j) FK IN COL 512 \u2013 \u2013 82K 52.18 69.5 52.1 47.5 64.0 24.6 49.8\n(k) FK IN 512 COL+ (x,y) \u2013 166K 66.37 82.9 70.1 67.0 77.0 36.1 70.0\n(l) FK IN 512 COL+ (x,y) (C) f s 166K 67.93 85.1 70.5 67.5 77.4 35.7 71.2\n(m) CNN F \u2013 (C) f s 4K 77.38 88.7 83.9 87.0 84.7 46.9 77.5\n(n) CNN S \u2013 (C) f s 4K 79.74 90.7 85.7 88.9 86.6 50.5 80.1\n(o) CNN M \u2013 \u2013 4K 76.97 89.5 84.3 88.8 83.2 48.4 77.0\n(p) CNN M \u2013 (C) f s 4K 79.89 91.7 85.4 89.5 86.6 51.6 79.3\n(q) CNN M \u2013 (C) f m 4K 79.50 90.9 84.6 89.4 85.8 50.3 78.4"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a187f-6db4-11ee-a9db-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-30", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(q) CNN M \u2013 (C) f m 4K 79.50 90.9 84.6 89.4 85.8 50.3 78.4\n(r) CNN M \u2013 (C) s s 4K 79.44 91.4 85.2 89.1 86.1 52.1 78.0\n(s) CNN M \u2013 (C) t t 41K 78.77 90.7 85.0 89.2 85.8 51.0 77.8\n(t) CNN M \u2013 (C) f - 4K 77.78 90.5 84.3 88.8 84.5 47.9 78.0\n(u) CNN M \u2013 (F) f - 4K 76.99 90.1 84.2 89.0 83.5 48.1 77.2\n(v) CNN M GS \u2013 \u2013 4K 73.59 87.4 80.8 82.4 82.1 44.5 73.5\n(w) CNN M GS \u2013 (C) f s 4K 77.00 89.4 83.8 85.1 84.4 49.4 77.6\n(x) CNN M 2048 \u2013 (C) f s 2K 80.10 91.3 85.8 89.9 86.7 52.4 79.7"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1880-6db4-11ee-9742-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-31", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(x) CNN M 2048 \u2013 (C) f s 2K 80.10 91.3 85.8 89.9 86.7 52.4 79.7\n(y) CNN M 1024 \u2013 (C) f s 1K 79.91 91.4 86.9 89.3 85.8 53.3 79.8\n(z) CNN M 128 \u2013 (C) f s 128 78.60 91.3 83.9 89.2 86.9 52.1 81.0\n(a) FK+CNN F (x,y) (C) f s 88K 77.95 89.6 83.1 87.1 84.5 48.0 79.4\n(b) FK+CNN M 2048 (x,y) (C) f s 86K 80.14 90.9 85.9 88.8 85.5 52.3 81.4\n(g) CNN S TUNE-RNK \u2013 (C) f s 4K 82.42 95.3 90.4 92.5 89.6 54.4 81.9\nTABLE 2\nVOC 2007 results (continued overleaf) . See Sect. 4 for details.\ndiscussed above, by a large \u001810% mAP margin\n([i]vs. [p]). Our small architecture CNN-F, which is"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1881-6db4-11ee-bb27-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-32", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "discussed above, by a large \u001810% mAP margin\n([i]vs. [p]). Our small architecture CNN-F, which is\nsimilar to DeCAF [7], performs signi\ufb01cantly better\nthan the latter ([II] vs. [s]), validating our implementation. Both medium CNN-M [m] and slow\nCNN-S [p] outperform the fast CNN-F [m] by a\nsigni\ufb01cant 2\u00183% margin. Since the accuracy\nof CNN-S and CNN-M is nearly the same, we\nfocus on the latter as it is simpler and marginally\n(\u001825%) faster. Remarkably, these good networks\nwork very well even with no augmentation [o].\nAnother advantage of CNNs compared to IFV is\nthe small dimensionality of the output features,although IFV can be compressed to an extent. We\nexplored retraining the CNNs such that the \ufb01nal\nlayer was of a lower dimensionality, and reducing\nfrom 4096 to 2048 actually resulted in a marginal\nperformance boost ([x] vs. [p]). What is surprising\nis that we can reduce the output dimensionality\nfurther to 1024D [y] and even 128D [z] with only\na drop of\u00182%for codes that are 32\u0002smaller"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1882-6db4-11ee-af9e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-33", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "further to 1024D [y] and even 128D [z] with only\na drop of\u00182%for codes that are 32\u0002smaller\n(\u0018650\u0002smaller than our best performing IFV [i]).\nNote,`2-normalising the features accounted for up\nto\u00185%of their performance over VOC 2007; it\nshould be applied before input to the SVM and\nafter pooling the augmented descriptors (where\n8\n(I) 79.9 61.4 56.0 49.6 58.4 44.8 78.8 70.8 85.0 31.7 51.0 56.4 80.2 57.5\n(II) 83.7 83.7 54.3 61.9 70.2 79.5 85.3 77.2 90.9 51.1 73.8 57.0 86.4 68.0\n(a) 80.7 64.4 53.8 53.8 60.2 47.8 79.9 68.9 86.1 37.3 51.1 55.8 83.7 56.9\n(b) 80.6 64.8 53.9 54.9 60.7 50.5 80.4 69.5 86.2 38.3 54.4 56.3 82.7 56.7"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1883-6db4-11ee-b506-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-34", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(c) 81.1 64.2 51.1 53.4 61.9 50.0 80.0 67.5 85.3 35.7 51.9 53.8 83.5 58.9\n(d) 80.5 66.9 50.9 53.9 62.1 51.5 80.5 68.5 85.9 37.2 55.2 54.3 83.3 59.2\n(e) 80.5 66.8 51.0 54.1 62.2 51.5 80.4 68.2 86.0 37.3 55.1 54.2 83.3 59.2\n(f) 82.4 71.6 52.8 62.4 63.4 57.1 81.6 70.9 86.9 41.2 61.2 56.9 85.2 61.5\n(g) 81.9 71.0 52.8 61.6 62.2 56.8 81.8 70.0 86.5 41.5 61.0 56.5 84.3 60.9\n(h) 81.1 67.9 52.6 55.4 61.4 51.2 80.5 69.1 86.4 41.2 56.0 56.2 83.7 59.9"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1884-6db4-11ee-bc1a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-35", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(i) 82.5 73.2 54.7 62.7 64.5 56.6 82.2 71.3 87.5 43.0 62.0 59.3 85.7 62.4\n(j) 66.1 46.6 42.5 35.8 41.1 45.5 75.4 58.3 83.9 39.8 47.3 35.6 69.2 49.0\n(k) 80.0 65.9 52.8 56.1 61.0 56.9 81.4 69.6 88.4 49.0 59.2 56.4 84.7 62.8\n(l) 81.6 70.8 52.9 59.6 63.1 59.9 82.1 70.5 88.9 50.6 63.7 57.5 86.1 64.1\n(m) 86.3 85.4 58.6 71.0 72.6 82.0 87.9 80.7 91.8 58.5 77.4 66.3 89.1 71.3\n(n) 87.8 88.3 61.3 74.8 74.7 87.2 89.0 83.7 92.3 58.8 80.5 69.4 90.5 74.0"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1885-6db4-11ee-b00e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-36", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(o) 85.1 87.4 58.1 70.4 73.1 83.5 85.5 80.9 90.8 54.1 78.9 61.1 89.0 70.4\n(p) 87.7 88.6 60.3 80.1 74.4 85.9 88.2 84.6 92.1 60.3 80.5 66.2 91.3 73.5\n(q) 87.6 88.6 60.7 78.2 73.6 86.0 87.4 83.8 92.3 59.3 81.0 66.8 91.3 74.0\n(r) 87.5 88.1 60.4 76.9 74.8 85.8 88.1 84.3 92.2 59.5 79.3 65.8 90.8 73.5\n(s) 87.3 87.6 60.1 72.3 75.3 85.2 86.9 82.6 91.9 58.5 77.9 66.5 90.5 73.4\n(t) 85.7 87.9 58.3 74.2 73.9 84.7 86.6 82.0 91.0 55.8 79.2 62.1 89.3 71.0"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1886-6db4-11ee-9bfe-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-37", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(u) 85.3 87.3 58.1 70.0 73.4 83.5 86.0 80.8 90.9 53.9 78.1 61.2 88.8 70.6\n(v) 85.0 84.9 57.8 65.9 69.8 79.5 82.9 77.4 89.2 42.8 71.7 60.2 86.3 67.8\n(w) 87.2 86.5 59.5 72.4 74.1 81.7 86.0 82.3 90.8 48.9 73.7 66.8 89.6 71.0\n(x) 87.6 88.4 60.2 76.9 75.4 85.5 88.0 83.4 92.1 61.1 83.1 68.5 91.9 74.2\n(y) 87.8 88.6 59.0 77.2 73.1 85.9 88.3 83.5 91.8 59.9 81.4 68.3 93.0 74.1\n(z) 86.6 87.5 59.1 70.0 72.9 84.6 86.7 83.6 89.4 57.0 81.5 64.8 90.4 73.4"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1887-6db4-11ee-861e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-38", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(a) 86.8 85.6 59.9 72.0 73.4 81.4 88.6 80.5 92.1 60.6 77.3 66.4 89.3 73.3\n(b) 87.7 88.4 61.2 76.9 76.6 84.9 89.1 82.9 92.4 61.9 80.9 68.7 91.5 75.1\n(g) 91.5 91.9 64.1 76.3 74.9 89.7 92.2 86.9 95.2 60.7 82.9 68.0 95.5 74.4\nTABLE 2\nVOC 2007 results (continued from previous page)\napplicable).\n4.5 Scenario 3: Deep representation (CNN) with\npre-training and \ufb01ne-tuning\nWe \ufb01ne-tuned our CNN-S architecture on VOC2007 using the ranking hinge loss, and achieved\na signi\ufb01cant improvement: 2:7%([g]vs. [n]). This\ndemonstrates that in spite of the small amount\nof VOC training data (5,011 images), \ufb01ne-tuning\nis able to adjust the learnt deep representation to\nbetter suit the dataset in question.4.6 Combinations\nFor the CNN-M 2048 representation [x], stacking"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1888-6db4-11ee-a8ba-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-39", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "is able to adjust the learnt deep representation to\nbetter suit the dataset in question.4.6 Combinations\nFor the CNN-M 2048 representation [x], stacking\ndeep and shallow representations to form a higherdimensional descriptor makes little difference ([x]\nvs. [b]). For the weaker CNN-F it results in a small\nboost of\u00180:8%([m] vs. [a]).\n4.7 Comparison with the state of the art\nIn Table 3 we report our results on ILSVRC-2012,\nVOC-2007, VOC-2012, Caltech-101, and Caltech-256\ndatasets, and compare them to the state of the art.\nFirst, we note that the ILSVRC error rates of our\nCNN-F, CNN-M, and CNN-S networks are better\n9\nILSVRC-2012 VOC-2007 VOC-2012 Caltech-101 Caltech-256\n(top-5 error) (mAP) (mAP) (accuracy) (accuracy)\n(a) FK IN 512 - 68.0 \u2013 \u2013 \u2013\n(b) CNN F 16.7 77.4 79.9 \u2013 \u2013\n(c) CNN M 13.7 79.9 82.5 87.15 \u00060.80 77.03\u00060.46"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1889-6db4-11ee-bacc-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-40", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(c) CNN M 13.7 79.9 82.5 87.15 \u00060.80 77.03\u00060.46\n(d) CNN M 2048 13.5 80.1 82.4 86.64 \u00060.53 76.88\u00060.35\n(e) CNN S 13.1 79.7 82.9 87.76 \u00060.66 77.61\u00060.12\n(f) CNN S TUNE-CLS 13.1 - 83.0 88.35\u00060.56 77.33\u00060.56\n(g) CNN S TUNE-RNK 13.1 82.4 83.2 \u2013 \u2013\n(h) Zeiler & Fergus [19] 16.1 - 79.0 86.5 \u00060.5 74.2\u00060.3\n(i) Razavian et al. [9], [10] 14.7 77.2 \u2013 \u2013 \u2013\n(j) Oquab et al. [8] 18 77.7 78.7 (82.8*) \u2013 \u2013\n(k) Oquab et al. [16] - - 86.3*\u2013 \u2013\n(l) Wei et al. [17] - 81.5 ( 85.2*) 81.7 ( 90.3*) \u2013 \u2013"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a188a-6db4-11ee-bda2-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-41", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "(l) Wei et al. [17] - 81.5 ( 85.2*) 81.7 ( 90.3*) \u2013 \u2013\n(m) He et al. [29] 13.6 80.1 - 91.4\u00060.7 \u2013\nTABLE 3\nComparison with the state of the art on ILSVRC2012, VOC2007, VOC2012, Caltech-101, and\nCaltech-256. Results marked with * were achieved using models pre-trained on the extended ILSVRC\ndatasets (1512 classes in [8], [16], 2000 classes in [17]). All other results were achieved using CNNs\npre-trained on ILSVRC-2012 (1000 classes).\nthan those reported by [13], [19], and [10] for the\nrelated con\ufb01gurations. This validates our implementation, and the difference is likely to be due to\nthe sampling of image crops from the uncropped\nimage plane (instead of the centre). When using\nour CNN features on other datasets, the relative\nperformance generally follows the same pattern\nas on ILSVRC, where the nets are trained \u2013 the\nCNN-F architecture exhibits the worst performance,\nwith CNN-M and CNN-S performing considerably\nbetter.\nFurther \ufb01ne-tuning of CNN-S on the VOC"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a188b-6db4-11ee-a7f3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-42", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "CNN-F architecture exhibits the worst performance,\nwith CNN-M and CNN-S performing considerably\nbetter.\nFurther \ufb01ne-tuning of CNN-S on the VOC\ndatasets turns out to be bene\ufb01cial; on VOC-2012,\nusing the ranking loss is marginally better than\nthe classi\ufb01cation loss ([g] vs. [f]), which can be\nexplained by the ranking-based VOC evaluation\ncriterion. Fine-tuning on Caltech-101 also yields a\nsmall improvement, but no gain is observed over\nCaltech-256.\nOur CNN-S net is competitive with recent CNNbased approaches [19], [9], [8], [16], [17], [29] and\non a number of datasets (VOC-2007, VOC-2012,\nCaltech-101, Caltech-256) and sets the state of the\nart on VOC-2007 and VOC-2012 across methods\npre-trained solely on ILSVRC-2012 dataset. While\nthe CNN-based methods of [16], [17] achieve better"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a188c-6db4-11ee-b1c4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-43", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "pre-trained solely on ILSVRC-2012 dataset. While\nthe CNN-based methods of [16], [17] achieve better\nperformance on VOC (86.3% and 90.3% respectively), they were trained using extended ILSVRCdatasets, enriched with additional categories semantically close to the ones in VOC. Additionally, [17] used a signi\ufb01cantly more complex classi\ufb01cation pipeline, driven by bounding box proposals [30], pre-trained on ILSVRC-2013 detection\ndataset. Their best reported result on VOC-2012\n(90.3%) was achieved by the late fusion with a\ncomplex hand-crafted method of [31]; without fusion, they get 84.2%. On Caltech-101, [29] achieves\nthe state of the art using spatial pyramid pooling\nof conv5 layer features, while we used full7 layer\nfeatures consistently across all datasets (for full7\nfeatures, they report 87:08%).\nIn addition to achieving performance comparable\nto the state of the art with a very simple approach\n(but powerful CNN-based features), with the modi\ufb01cations outlined in the paper (primarily the use\nof data augmentation similar to the CNN-based\nmethods) we are able to improve the performance"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a188d-6db4-11ee-abac-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-44", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "of data augmentation similar to the CNN-based\nmethods) we are able to improve the performance\nof shallow IFV to 68.02% (Table 2, [i]).\n4.8 Performance Evolution on VOC-2007\nA comparative plot of the evolution in the performance of the methods evaluated in this paper,\nalong with a selection from our earlier review of\nshallow methods [12] is presented in Fig. 1. Classi\ufb01cation accuracy over PASCAL VOC was 54.48%\nmAP for the BoVW model in 2008, 61.7% for the\n10\n      BOW\n32K\n\u2013FK-BL\n327K\n\u2013\n[I]FK\n327K\n\u2013\n[a]FK-IN\n84K\nf s\n[i]DeCAF\n4K\nt t\n[II]CNN-F\n4K\nf s\n[m]CNN-M 2K\n2K\nf s\n[y]CNN-S\n4K (TN)\nf s\n[\u03b3]   \n                           \n545658606264666870727476788082mAP 68.02\n54.4861.6963.6673.4177.1580.132008 2010 2013 2014 ...\n82.42\nMethod\nDim.\nAug.\nRef."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a188e-6db4-11ee-ab0f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-45", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "54.4861.6963.6673.4177.1580.132008 2010 2013 2014 ...\n82.42\nMethod\nDim.\nAug.\nRef.\nFig. 1. Evolution of Performance on PASCAL VOC-2007 over the recent years. Please refer to Table 2\nfor details and references.\nIFV in 2010 [12], and 73.41% for DeCAF [7] and\nsimilar [8], [9] CNN-based methods introduced in\nlate 2013. Our best performing CNN-based method\n(CNN-S with \ufb01ne-tuning) achieves 82.42%, comparable to the most recent state-of-the-art.\n4.9 Timings and dimensionality\nOne of our best-performing CNN representations\nCNN-M-2048 [x] is \u001842\u0002more compact than the\nbest performing IFV [i] (84K vs. 2K) and CNN-M\nfeatures are also \u001850\u0002faster to compute ( \u0018120s\nvs.\u00182:4sper image with augmentation enabled,\nover a single CPU core). Non-augmented CNN-M\nfeatures [o] take around 0:3sper image, compared\nto\u00180:4sfor CNN-S features and \u00180:13sfor CNNF features.\n5 C ONCLUSION"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a188f-6db4-11ee-a6a3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-46", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "to\u00180:4sfor CNN-S features and \u00180:13sfor CNNF features.\n5 C ONCLUSION\nIn this paper we presented a rigorous empirical evaluation of CNN-based methods for image\nclassi\ufb01cation, along with a comparison with more\ntraditional shallow feature encoding methods. We\nhave demonstrated that the performance of shallow representations can be signi\ufb01cantly improved\nby adopting data augmentation, typically used indeep learning. In spite of this improvement, deep\narchitectures still outperform the shallow methods\nby a large margin. We have shown that the performance of deep representations on the ILSVRC\ndataset is a good indicator of their performance\non other datasets, and that \ufb01ne-tuning can further\nimprove on already very strong results achieved\nusing the combination of deep representations and\na linear SVM. Source code and CNN models to\nreproduce the experiments presented in the paper\nare available on the project website [32] in the\nhope that it would provide common ground for\nfuture comparisons, and good baselines for image\nrepresentation research.\nACKNOWLEDGEMENTS\nThis work was supported by the EPSRC and ERC\ngrant VisRec no. 228180. We gratefully acknowledge the support of NVIDIA Corporation with the\ndonation of the GPUs used for this research.\nREFERENCES"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1890-6db4-11ee-89c4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-47", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "grant VisRec no. 228180. We gratefully acknowledge the support of NVIDIA Corporation with the\ndonation of the GPUs used for this research.\nREFERENCES\n[1] G. Csurka, C. Bray, C. Dance, and L. Fan, \u201cVisual categorization with bags of keypoints,\u201d in Workshop on Statistical\nLearning in Computer Vision, ECCV , 2004, pp. 1\u201322.\n11\n[2] J. Sivic and A. Zisserman, \u201cVideo Google: A text retrieval\napproach to object matching in videos,\u201d in Proc. ICCV ,\nvol. 2, 2003, pp. 1470\u20131477.\n[3] F. Perronnin, J. S \u00b4anchez, and T. Mensink, \u201cImproving the\nFisher kernel for large-scale image classi\ufb01cation,\u201d in Proc.\nECCV , 2010.\n[4] Y. LeCun, B. Boser, J. S. Denker, D. Henderson, R. E.\nHoward, W. Hubbard, and L. D. Jackel, \u201cBackpropagation applied to handwritten zip code recognition,\u201d Neural\nComputation , vol. 1, no. 4, pp. 541\u2013551, 1989."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1891-6db4-11ee-a1eb-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-48", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "Computation , vol. 1, no. 4, pp. 541\u2013551, 1989.\n[5] J. Deng, W. Dong, R. Socher, L.-J. Li, K. Li, and L. Fei-Fei,\n\u201cImagenet: A large-scale hierarchical image database,\u201d in\nProc. CVPR , 2009.\n[6] M. Everingham, L. Van Gool, C. K. I. Williams, J. Winn,\nand A. Zisserman, \u201cThe PASCAL Visual Object Classes\n(VOC) challenge,\u201d IJCV , vol. 88, no. 2, pp. 303\u2013338, 2010.\n[7] J. Donahue, Y. Jia, O. Vinyals, J. Hoffman, N. Zhang,\nE. Tzeng, and T. Darrell, \u201cDecaf: A deep convolutional\nactivation feature for generic visual recognition,\u201d CoRR ,\nvol. abs/1310.1531, 2013.\n[8] M. Oquab, L. Bottou, I. Laptev, and J. Sivic, \u201cLearning\nand Transferring Mid-Level Image Representations using\nConvolutional Neural Networks,\u201d in Proc. CVPR , 2014."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1892-6db4-11ee-aa31-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-49", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "and Transferring Mid-Level Image Representations using\nConvolutional Neural Networks,\u201d in Proc. CVPR , 2014.\n[9] A. Razavian, H. Azizpour, J. Sullivan, and S. Carlsson,\n\u201cCNN Features off-the-shelf: an Astounding Baseline for\nRecognition,\u201d CoRR , vol. abs/1403.6382, 2014.\n[10] P . Sermanet, D. Eigen, X. Zhang, M. Mathieu, R. Fergus,\nand Y. LeCun, \u201cOverFeat: Integrated Recognition, Localization and Detection using Convolutional Networks,\u201d in\nProc. ICLR , 2014.\n[11] R. B. Girshick, J. Donahue, T. Darrell, and J. Malik, \u201cRich\nfeature hierarchies for accurate object detection and semantic segmentation,\u201d in Proc. CVPR , 2014.\n[12] K. Chat\ufb01eld, V . Lempitsky, A. Vedaldi, and A. Zisserman,\n\u201cThe devil is in the details: an evaluation of recent feature\nencoding methods,\u201d in Proc. BMVC. , 2011."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1893-6db4-11ee-8d0f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-50", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "\u201cThe devil is in the details: an evaluation of recent feature\nencoding methods,\u201d in Proc. BMVC. , 2011.\n[13] A. Krizhevsky, I. Sutskever, and G. E. Hinton, \u201cImageNet\nclassi\ufb01cation with deep convolutional neural networks,\u201d\ninNIPS , 2012, pp. 1106\u20131114.\n[14] M. Paulin, J. Revaud, Z. Harchaoui, F. Perronnin, and\nC. Schmid, \u201cTransformation Pursuit for Image Classi\ufb01cation,\u201d in Proc. CVPR , 2014.\n[15] F. Perronnin, Z. Akata, Z. Harchaoui, and C. Schmid,\n\u201cTowards good practice in large-scale learning for image\nclassi\ufb01cation,\u201d in Proc. CVPR , 2012, pp. 3482\u20133489.\n[16] M. Oquab, L. Bottou, I. Laptev, and J. Sivic, \u201cWeakly\nsupervised object recognition with convolutional neural\nnetworks,\u201d INRIA, Tech. Rep. HAL-01015140, 2014."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1894-6db4-11ee-b07e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-51", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "supervised object recognition with convolutional neural\nnetworks,\u201d INRIA, Tech. Rep. HAL-01015140, 2014.\n[17] Y. Wei, W. Xia, J. Huang, B. Ni, J. Dong, Y. Zhao, and\nS. Yan, \u201cCNN: Single-label to multi-label,\u201d CoRR , vol.\nabs/1406.5726, 2014.\n[18] H. J \u00b4egou, F. Perronnin, M. Douze, J. S \u00b4anchez, P . P \u00b4erez, and\nC. Schmid, \u201cAggregating local images descriptors into\ncompact codes,\u201d IEEE P AMI , 2012.\n[19] M. D. Zeiler and R. Fergus, \u201cVisualizing and understanding convolutional networks,\u201d CoRR , vol. abs/1311.2901,\n2013.\n[20] Y. Jia, \u201cCaffe: An open source convolutional architecture\nfor fast feature embedding,\u201d http://caffe.berkeleyvision.\norg/, 2013.\n[21] L. Fei-Fei, R. Fergus, and P . Perona, \u201cLearning generative"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1895-6db4-11ee-a378-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-52", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "org/, 2013.\n[21] L. Fei-Fei, R. Fergus, and P . Perona, \u201cLearning generative\nvisual models from few training examples: An incremental bayesian approach tested on 101 object categories,\u201d\ninIEEE CVPR Workshop of Generative Model Based Vision ,\n2004.[22] G. Grif\ufb01n, A. Holub, and P . Perona, \u201cCaltech-256 object\ncategory dataset,\u201d California Institute of Technology,\nTech. Rep. 7694, 2007. [Online]. Available: http://\nauthors.library.caltech.edu/7694\n[23] J. S \u00b4anchez, F. Perronnin, and T. Em \u00b4\u0131dio de Campos,\n\u201cModeling the spatial layout of images beyond spatial\npyramids,\u201d Pattern Recognition Letters , vol. 33, no. 16, pp.\n2216\u20132223, 2012.\n[24] R. Arandjelovi \u00b4c and A. Zisserman, \u201cThree things everyone should know to improve object retrieval,\u201d in Proc.\nCVPR , 2012.\n[25] A. Vedaldi and A. Zisserman, \u201cEf\ufb01cient additive kernels\nvia explicit feature maps,\u201d IEEE P AMI , 2011."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1896-6db4-11ee-aed5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-53", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "[25] A. Vedaldi and A. Zisserman, \u201cEf\ufb01cient additive kernels\nvia explicit feature maps,\u201d IEEE P AMI , 2011.\n[26] S. Lazebnik, C. Schmid, and J. Ponce, \u201cBeyond Bags\nof Features: Spatial Pyramid Matching for Recognizing\nNatural Scene Categories,\u201d in Proc. CVPR , 2006.\n[27] R. Arandjelovi \u00b4c and A. Zisserman, \u201cAll about VLAD,\u201d in\nProc. CVPR , 2013.\n[28] H. J \u00b4egou, M. Douze, and C. Schmid, \u201cOn the burstiness\nof visual elements,\u201d in Proc. CVPR , Jun 2009.\n[29] K. He, A. Zhang, S. Ren, and J. Sun, \u201cSpatial pyramid\npooling in deep convolutional networks for visual recognition,\u201d in Proc. ECCV , 2014.\n[30] M.-M. Cheng, Z. Zhang, W.-Y. Lin, and P . H. S. Torr,\n\u201cBING: Binarized normed gradients for objectness estimation at 300fps,\u201d in Proc. CVPR , 2014."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1897-6db4-11ee-8bd3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1405.3531-54", "title": "Return of the Devil in the Details: Delving Deep into Convolutional Nets", "source": "http://arxiv.org/pdf/1405.3531", "start_index": 0, "chroma:document": "\u201cBING: Binarized normed gradients for objectness estimation at 300fps,\u201d in Proc. CVPR , 2014.\n[31] S. Yan, J. Dong, Q. Chen, Z. Song, Y. Pan, W. Xia,\nH. Zhongyang, Y. Hua, and S. Shen, \u201cGeneralized hierarchical matching for subcategory aware object classi\ufb01cation,\u201d in The P ASCAL Visual Object Classes Challenge\nWorkshop , 2012.\n[32] K. Chat\ufb01eld, K. Simonyan, A. Vedaldi, and A. Zisserman,\n\u201cReturn of the devil in the details: delving deep into\nconvolutional nets webpage,\u201d 2014. [Online]. Available:\nhttp://www.robots.ox.ac.uk/\u0018vgg/research/deep eval"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1898-6db4-11ee-b46e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-0", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Learning Phrase Representations using RNN Encoder\u2013Decoder\nfor Statistical Machine Translation\nKyunghyun Cho\nBart van Merri \u00a8enboer Caglar Gulcehre\nUniversit \u00b4e de Montr \u00b4eal\nfirstname.lastname@umontreal.caDzmitry Bahdanau\nJacobs University, Germany\nd.bahdanau@jacobs-university.de\nFethi Bougares Holger Schwenk\nUniversit \u00b4e du Maine, France\nfirstname.lastname@lium.univ-lemans.frYoshua Bengio\nUniversit \u00b4e de Montr \u00b4eal, CIFAR Senior Fellow\nfind.me@on.the.web\nAbstract\nIn this paper, we propose a novel neural network model called RNN Encoder\u2013\nDecoder that consists of two recurrent\nneural networks (RNN). One RNN encodes a sequence of symbols into a \ufb01xedlength vector representation, and the other\ndecodes the representation into another sequence of symbols. The encoder and decoder of the proposed model are jointly\ntrained to maximize the conditional probability of a target sequence given a source"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a1899-6db4-11ee-b242-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-1", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "decodes the representation into another sequence of symbols. The encoder and decoder of the proposed model are jointly\ntrained to maximize the conditional probability of a target sequence given a source\nsequence. The performance of a statistical machine translation system is empirically found to improve by using the conditional probabilities of phrase pairs computed by the RNN Encoder\u2013Decoder as an\nadditional feature in the existing log-linear\nmodel. Qualitatively, we show that the\nproposed model learns a semantically and\nsyntactically meaningful representation of\nlinguistic phrases.\n1 Introduction\nDeep neural networks have shown great success in\nvarious applications such as objection recognition\n(see, e.g., (Krizhevsky et al., 2012)) and speech\nrecognition (see, e.g., (Dahl et al., 2012)). Furthermore, many recent works showed that neural networks can be successfully used in a number of tasks in natural language processing (NLP).\nThese include, but are not limited to, language\nmodeling (Bengio et al., 2003), paraphrase detection (Socher et al., 2011) and word embedding extraction (Mikolov et al., 2013). In the \ufb01eld of statistical machine translation (SMT), deep neural\nnetworks have begun to show promising results.\n(Schwenk, 2012) summarizes a successful usage\nof feedforward neural networks in the framework"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a189a-6db4-11ee-b872-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-2", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "networks have begun to show promising results.\n(Schwenk, 2012) summarizes a successful usage\nof feedforward neural networks in the framework\nof phrase-based SMT system.Along this line of research on using neural networks for SMT, this paper focuses on a novel neural network architecture that can be used as a part\nof the conventional phrase-based SMT system.\nThe proposed neural network architecture, which\nwe will refer to as an RNN Encoder\u2013Decoder , consists of two recurrent neural networks (RNN) that\nact as an encoder and a decoder pair. The encoder maps a variable-length source sequence to a\n\ufb01xed-length vector, and the decoder maps the vector representation back to a variable-length target\nsequence. The two networks are trained jointly to\nmaximize the conditional probability of the target\nsequence given a source sequence. Additionally,\nwe propose to use a rather sophisticated hidden\nunit in order to improve both the memory capacity\nand the ease of training.\nThe proposed RNN Encoder\u2013Decoder with a\nnovel hidden unit is empirically evaluated on the\ntask of translating from English to French. We\ntrain the model to learn the translation probability of an English phrase to a corresponding French\nphrase. The model is then used as a part of a standard phrase-based SMT system by scoring each"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a189b-6db4-11ee-9cc8-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-3", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "train the model to learn the translation probability of an English phrase to a corresponding French\nphrase. The model is then used as a part of a standard phrase-based SMT system by scoring each\nphrase pair in the phrase table. The empirical evaluation reveals that this approach of scoring phrase\npairs with an RNN Encoder\u2013Decoder improves\nthe translation performance.\nWe qualitatively analyze the trained RNN\nEncoder\u2013Decoder by comparing its phrase scores\nwith those given by the existing translation model.\nThe qualitative analysis shows that the RNN\nEncoder\u2013Decoder is better at capturing the linguistic regularities in the phrase table, indirectly\nexplaining the quantitative improvements in the\noverall translation performance. The further analysis of the model reveals that the RNN Encoder\u2013\nDecoder learns a continuous space representation\nof a phrase that preserves both the semantic and\nsyntactic structure of the phrase.arXiv:1406.1078v3  [cs.CL]  3 Sep 2014\n2 RNN Encoder\u2013Decoder\n2.1 Preliminary: Recurrent Neural Networks\nA recurrent neural network (RNN) is a neural network that consists of a hidden state hand an\noptional output ywhich operates on a variablelength sequence x= (x1;:::;x T). At each time\nstept, the hidden state hhtiof the RNN is updated\nby"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a189c-6db4-11ee-a854-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-4", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "stept, the hidden state hhtiof the RNN is updated\nby\nhhti=f\u0000\nhht\u00001i;xt\u0001\n; (1)\nwherefis a non-linear activation function.fmay be as simple as an elementwise logistic sigmoid function and as complex as a long short-term memory (LSTM)\nunit (Hochreiter and Schmidhuber, 1997).\nAn RNN can learn a probability distribution\nover a sequence by being trained to predict the\nnext symbol in a sequence. In that case, the output\nat each timestep tis the conditional distribution\np(xtjxt\u00001;:::;x 1). For example, a multinomial\ndistribution ( 1-of-Kcoding) can be output using a\nsoftmax activation function\np(xt;j= 1jxt\u00001;:::;x 1) =exp\u0000\nwjhhti\u0001\nPK\nj0=1exp\u0000\nwj0hhti\u0001;\n(2)\nfor all possible symbols j= 1;:::;K , where wj\nare the rows of a weight matrix W. By combining\nthese probabilities, we can compute the probability of the sequence xusing\np(x) =TY"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a189d-6db4-11ee-af09-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-5", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "are the rows of a weight matrix W. By combining\nthese probabilities, we can compute the probability of the sequence xusing\np(x) =TY\nt=1p(xtjxt\u00001;:::;x 1): (3)\nFrom this learned distribution, it is straightforward to sample a new sequence by iteratively sampling a symbol at each time step.\n2.2 RNN Encoder\u2013Decoder\nIn this paper, we propose a novel neural network\narchitecture that learns to encode a variable-length\nsequence into a \ufb01xed-length vector representation\nand to decode a given \ufb01xed-length vector representation back into a variable-length sequence.\nFrom a probabilistic perspective, this new model\nis a general method to learn the conditional distribution over a variable-length sequence conditioned on yet another variable-length sequence,\ne.g.p(y1;:::;y T0jx1;:::;x T), where one\nx1x2 xTyT' y2 y1\ncDecoder\nEncoderFigure 1: An illustration of the proposed RNN\nEncoder\u2013Decoder.\nshould note that the input and output sequence\nlengthsTandT0may differ.\nThe encoder is an RNN that reads each symbol\nof an input sequence xsequentially. As it reads"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a189e-6db4-11ee-9042-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-6", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "lengthsTandT0may differ.\nThe encoder is an RNN that reads each symbol\nof an input sequence xsequentially. As it reads\neach symbol, the hidden state of the RNN changes\naccording to Eq. (1). After reading the end of\nthe sequence (marked by an end-of-sequence symbol), the hidden state of the RNN is a summary c\nof the whole input sequence.\nThe decoder of the proposed model is another\nRNN which is trained to generate the output sequence by predicting the next symbol ytgiven the\nhidden state hhti. However, unlike the RNN described in Sec. 2.1, both ytandhhtiare also conditioned onyt\u00001and on the summary cof the input\nsequence. Hence, the hidden state of the decoder\nat timetis computed by,\nhhti=f\u0000\nhht\u00001i;yt\u00001;c\u0001\n;\nand similarly, the conditional distribution of the\nnext symbol is\nP(ytjyt\u00001;yt\u00002;:::;y 1;c) =g\u0000\nhhti;yt\u00001;c\u0001\n:\nfor given activation functions fandg(the latter\nmust produce valid probabilities, e.g. with a softmax).\nSee Fig. 1 for a graphical depiction of the proposed model architecture."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a189f-6db4-11ee-9074-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-7", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "for given activation functions fandg(the latter\nmust produce valid probabilities, e.g. with a softmax).\nSee Fig. 1 for a graphical depiction of the proposed model architecture.\nThe two components of the proposed RNN\nEncoder\u2013Decoder are jointly trained to maximize\nthe conditional log-likelihood\nmax\n\u00121\nNNX\nn=1logp\u0012(ynjxn); (4)\nwhere \u0012is the set of the model parameters and\neach (xn;yn)is an (input sequence, output sequence) pair from the training set. In our case,\nas the output of the decoder, starting from the input, is differentiable, we can use a gradient-based\nalgorithm to estimate the model parameters.\nOnce the RNN Encoder\u2013Decoder is trained, the\nmodel can be used in two ways. One way is to use\nthe model to generate a target sequence given an\ninput sequence. On the other hand, the model can\nbe used to score a given pair of input and output\nsequences, where the score is simply a probability\np\u0012(yjx)from Eqs. (3) and (4).\n2.3 Hidden Unit that Adaptively Remembers\nand Forgets\nIn addition to a novel model architecture, we also\npropose a new type of hidden unit ( fin Eq. (1))"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a0-6db4-11ee-95fd-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-8", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "and Forgets\nIn addition to a novel model architecture, we also\npropose a new type of hidden unit ( fin Eq. (1))\nthat has been motivated by the LSTM unit but is\nmuch simpler to compute and implement.1Fig. 2\nshows the graphical depiction of the proposed hidden unit.\nLet us describe how the activation of the j-th\nhidden unit is computed. First, the reset gaterjis\ncomputed by\nrj=\u001b\u0010\n[Wrx]j+\u0002\nUrhht\u00001i\u0003\nj\u0011\n; (5)\nwhere\u001bis the logistic sigmoid function, and [:]j\ndenotes the j-th element of a vector. xandht\u00001\nare the input and the previous hidden state, respectively. WrandUrare weight matrices which are\nlearned.\nSimilarly, the update gatezjis computed by\nzj=\u001b\u0010\n[Wzx]j+\u0002\nUzhht\u00001i\u0003\nj\u0011\n: (6)\nThe actual activation of the proposed unit hjis\nthen computed by\nhhti\nj=zjhht\u00001i\nj + (1\u0000zj)~hhti\nj; (7)\nwhere\n~hhti\nj=\u001e\u0010\n[Wx]j+\u0002\nU\u0000"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a1-6db4-11ee-bef1-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-9", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "j; (7)\nwhere\n~hhti\nj=\u001e\u0010\n[Wx]j+\u0002\nU\u0000\nr\fhht\u00001i\u0001\u0003\nj\u0011\n:(8)\nIn this formulation, when the reset gate is close\nto 0, the hidden state is forced to ignore the previous hidden state and reset with the current input\n1The LSTM unit, which has shown impressive results in\nseveral applications such as speech recognition, has a memory cell and four gating units that adaptively control the information \ufb02ow inside the unit, compared to only two gating\nunits in the proposed hidden unit. For details on LSTM networks, see, e.g., (Graves, 2012).\nz\nrh h~xFigure 2: An illustration of the proposed hidden\nactivation function. The update gate zselects\nwhether the hidden state is to be updated with\na new hidden state ~h. The reset gate rdecides\nwhether the previous hidden state is ignored. See\nEqs. (5)\u2013(8) for the detailed equations of r,z,h\nand~h.\nonly. This effectively allows the hidden state to\ndrop any information that is found to be irrelevant\nlater in the future, thus, allowing a more compact\nrepresentation.\nOn the other hand, the update gate controls how\nmuch information from the previous hidden state"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a2-6db4-11ee-b77c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-10", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "later in the future, thus, allowing a more compact\nrepresentation.\nOn the other hand, the update gate controls how\nmuch information from the previous hidden state\nwill carry over to the current hidden state. This\nacts similarly to the memory cell in the LSTM\nnetwork and helps the RNN to remember longterm information. Furthermore, this may be considered an adaptive variant of a leaky-integration\nunit (Bengio et al., 2013).\nAs each hidden unit has separate reset and update gates, each hidden unit will learn to capture\ndependencies over different time scales. Those\nunits that learn to capture short-term dependencies\nwill tend to have reset gates that are frequently active, but those that capture longer-term dependencies will have update gates that are mostly active.\nIn our preliminary experiments, we found that\nit is crucial to use this new unit with gating units.\nWe were not able to get meaningful result with an\noft-used tanh unit without any gating.\n3 Statistical Machine Translation\nIn a commonly used statistical machine translation\nsystem (SMT), the goal of the system (decoder,\nspeci\ufb01cally) is to \ufb01nd a translation fgiven a source\nsentence e, which maximizes\np(fje)/p(ejf)p(f);\nwhere the \ufb01rst term at the right hand side is called"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a3-6db4-11ee-9b43-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-11", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "sentence e, which maximizes\np(fje)/p(ejf)p(f);\nwhere the \ufb01rst term at the right hand side is called\ntranslation model and the latter language model\n(see, e.g., (Koehn, 2005)). In practice, however,\nmost SMT systems model logp(fje)as a loglinear model with additional features and corresponding weights:\nlogp(fje) =NX\nn=1wnfn(f;e) + logZ(e);(9)\nwherefnandwnare then-th feature and weight,\nrespectively. Z(e)is a normalization constant that\ndoes not depend on the weights. The weights are\noften optimized to maximize the BLEU score on a\ndevelopment set.\nIn the phrase-based SMT framework\nintroduced in (Koehn et al., 2003) and\n(Marcu and Wong, 2002), the translation model\nlogp(ejf)is factorized into the translation\nprobabilities of matching phrases in the source\nand target sentences.2These probabilities are\nonce again considered additional features in the\nlog-linear model (see Eq. (9)) and are weighted\naccordingly to maximize the BLEU score.\nSince the neural net language model was proposed in (Bengio et al., 2003), neural networks"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a4-6db4-11ee-9023-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-12", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "accordingly to maximize the BLEU score.\nSince the neural net language model was proposed in (Bengio et al., 2003), neural networks\nhave been used widely in SMT systems. In\nmany cases, neural networks have been used to\nrescore translation hypotheses ( n-best lists) (see,\ne.g., (Schwenk et al., 2006)). Recently, however,\nthere has been interest in training neural networks\nto score the translated sentence (or phrase pairs)\nusing a representation of the source sentence as\nan additional input. See, e.g., (Schwenk, 2012),\n(Son et al., 2012) and (Zou et al., 2013).\n3.1 Scoring Phrase Pairs with RNN\nEncoder\u2013Decoder\nHere we propose to train the RNN Encoder\u2013\nDecoder (see Sec. 2.2) on a table of phrase pairs\nand use its scores as additional features in the loglinear model in Eq. (9) when tuning the SMT decoder.\nWhen we train the RNN Encoder\u2013Decoder, we\nignore the (normalized) frequencies of each phrase\npair in the original corpora. This measure was\ntaken in order (1) to reduce the computational expense of randomly selecting phrase pairs from a"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a5-6db4-11ee-bb0d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-13", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "pair in the original corpora. This measure was\ntaken in order (1) to reduce the computational expense of randomly selecting phrase pairs from a\nlarge phrase table according to the normalized frequencies and (2) to ensure that the RNN Encoder\u2013\nDecoder does not simply learn to rank the phrase\npairs according to their numbers of occurrences.\nOne underlying reason for this choice was that the\nexisting translation probability in the phrase table already re\ufb02ects the frequencies of the phrase\n2Without loss of generality, from here on, we refer to\np(ejf)for each phrase pair as a translation model as wellpairs in the original corpus. With a \ufb01xed capacity\nof the RNN Encoder\u2013Decoder, we try to ensure\nthat most of the capacity of the model is focused\ntoward learning linguistic regularities, i.e., distinguishing between plausible and implausible translations, or learning the \u201cmanifold\u201d (region of probability concentration) of plausible translations.\nOnce the RNN Encoder\u2013Decoder is trained, we\nadd a new score for each phrase pair to the existing phrase table. This allows the new scores to enter into the existing tuning algorithm with minimal\nadditional overhead in computation.\nAs Schwenk pointed out in (Schwenk, 2012),\nit is possible to completely replace the existing"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a6-6db4-11ee-8956-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-14", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "additional overhead in computation.\nAs Schwenk pointed out in (Schwenk, 2012),\nit is possible to completely replace the existing\nphrase table with the proposed RNN Encoder\u2013\nDecoder. In that case, for a given source phrase,\nthe RNN Encoder\u2013Decoder will need to generate\na list of (good) target phrases. This requires, however, an expensive sampling procedure to be performed repeatedly. In this paper, thus, we only\nconsider rescoring the phrase pairs in the phrase\ntable.\n3.2 Related Approaches: Neural Networks in\nMachine Translation\nBefore presenting the empirical results, we discuss\na number of recent works that have proposed to\nuse neural networks in the context of SMT.\nSchwenk in (Schwenk, 2012) proposed a similar approach of scoring phrase pairs. Instead of the\nRNN-based neural network, he used a feedforward\nneural network that has \ufb01xed-size inputs (7 words\nin his case, with zero-padding for shorter phrases)\nand \ufb01xed-size outputs (7 words in the target language). When it is used speci\ufb01cally for scoring\nphrases for the SMT system, the maximum phrase\nlength is often chosen to be small. However, as the\nlength of phrases increases or as we apply neural"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a7-6db4-11ee-b9f9-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-15", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "phrases for the SMT system, the maximum phrase\nlength is often chosen to be small. However, as the\nlength of phrases increases or as we apply neural\nnetworks to other variable-length sequence data,\nit is important that the neural network can handle variable-length input and output. The proposed RNN Encoder\u2013Decoder is well-suited for\nthese applications.\nSimilar to (Schwenk, 2012), Devlin et al.\n(Devlin et al., 2014) proposed to use a feedforward neural network to model a translation model,\nhowever, by predicting one word in a target phrase\nat a time. They reported an impressive improvement, but their approach still requires the maximum length of the input phrase (or context words)\nto be \ufb01xed a priori.\nAlthough it is not exactly a neural network they\ntrain, the authors of (Zou et al., 2013) proposed\nto learn a bilingual embedding of words/phrases.\nThey use the learned embedding to compute the\ndistance between a pair of phrases which is used\nas an additional score of the phrase pair in an SMT\nsystem.\nIn (Chandar et al., 2014), a feedforward neural\nnetwork was trained to learn a mapping from a\nbag-of-words representation of an input phrase to\nan output phrase. This is closely related to both the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a8-6db4-11ee-b754-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-16", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "network was trained to learn a mapping from a\nbag-of-words representation of an input phrase to\nan output phrase. This is closely related to both the\nproposed RNN Encoder\u2013Decoder and the model\nproposed in (Schwenk, 2012), except that their input representation of a phrase is a bag-of-words.\nA similar approach of using bag-of-words representations was proposed in (Gao et al., 2013) as\nwell. Earlier, a similar encoder\u2013decoder model using two recursive neural networks was proposed\nin (Socher et al., 2011), but their model was restricted to a monolingual setting, i.e. the model\nreconstructs an input sentence. More recently, another encoder\u2013decoder model using an RNN was\nproposed in (Auli et al., 2013), where the decoder is conditioned on a representation of either\na source sentence or a source context.\nOne important difference between the proposed RNN Encoder\u2013Decoder and the approaches\nin (Zou et al., 2013) and (Chandar et al., 2014) is\nthat the order of the words in source and target phrases is taken into account. The RNN\nEncoder\u2013Decoder naturally distinguishes between\nsequences that have the same words but in a different order, whereas the aforementioned approaches\neffectively ignore order information.\nThe closest approach related to the proposed"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18a9-6db4-11ee-a178-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-17", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "sequences that have the same words but in a different order, whereas the aforementioned approaches\neffectively ignore order information.\nThe closest approach related to the proposed\nRNN Encoder\u2013Decoder is the Recurrent Continuous Translation Model (Model 2) proposed in\n(Kalchbrenner and Blunsom, 2013). In their paper, they proposed a similar model that consists\nof an encoder and decoder. The difference with\nour model is that they used a convolutional n-gram\nmodel (CGM) for the encoder and the hybrid of\nan inverse CGM and a recurrent neural network\nfor the decoder. They, however, evaluated their\nmodel on rescoring the n-best list proposed by the\nconventional SMT system and computing the perplexity of the gold standard translations.\n4 Experiments\nWe evaluate our approach on the English/French\ntranslation task of the WMT\u201914 workshop.4.1 Data and Baseline System\nLarge amounts of resources are available to build\nan English/French SMT system in the framework\nof the WMT\u201914 translation task. The bilingual\ncorpora include Europarl (61M words), news commentary (5.5M), UN (421M), and two crawled\ncorpora of 90M and 780M words respectively.\nThe last two corpora are quite noisy. To train"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18aa-6db4-11ee-92b3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-18", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "corpora of 90M and 780M words respectively.\nThe last two corpora are quite noisy. To train\nthe French language model, about 712M words of\ncrawled newspaper material is available in addition to the target side of the bitexts. All the word\ncounts refer to French words after tokenization.\nIt is commonly acknowledged that training statistical models on the concatenation of all this\ndata does not necessarily lead to optimal performance, and results in extremely large models which are dif\ufb01cult to handle. Instead, one\nshould focus on the most relevant subset of the\ndata for a given task. We have done so by\napplying the data selection method proposed in\n(Moore and Lewis, 2010), and its extension to bitexts (Axelrod et al., 2011). By these means we\nselected a subset of 418M words out of more\nthan 2G words for language modeling and a\nsubset of 348M out of 850M words for training the RNN Encoder\u2013Decoder. We used the\ntest set newstest2012 and 2013 for data\nselection and weight tuning with MERT, and\nnewstest2014 as our test set. Each set has\nmore than 70 thousand words and a single reference translation.\nFor training the neural networks, including the\nproposed RNN Encoder\u2013Decoder, we limited the\nsource and target vocabulary to the most frequent"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ab-6db4-11ee-b6bc-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-19", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "For training the neural networks, including the\nproposed RNN Encoder\u2013Decoder, we limited the\nsource and target vocabulary to the most frequent\n15,000 words for both English and French. This\ncovers approximately 93% of the dataset. All the\nout-of-vocabulary words were mapped to a special\ntoken ( [UNK ]).\nThe baseline phrase-based SMT system was\nbuilt using Moses with default settings. This system achieves a BLEU score of 30.64 and 33.3 on\nthe development and test sets, respectively (see Table 1).\n4.1.1 RNN Encoder\u2013Decoder\nThe RNN Encoder\u2013Decoder used in the experiment had 1000 hidden units with the proposed\ngates at the encoder and at the decoder. The input matrix between each input symbol xhtiand the\nhidden unit is approximated with two lower-rank\nmatrices, and the output matrix is approximated\nModelsBLEU\ndev test\nBaseline 30.64 33.30\nRNN 31.20 33.87\nCSLM + RNN 31.48 34.64\nCSLM + RNN + WP 31.50 34.54\nTable 1: BLEU scores computed on the development and test sets using different combinations of\napproaches. WP denotes a word penalty , where\nwe penalizes the number of unknown words to"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ac-6db4-11ee-815e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-20", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Table 1: BLEU scores computed on the development and test sets using different combinations of\napproaches. WP denotes a word penalty , where\nwe penalizes the number of unknown words to\nneural networks.\nsimilarly. We used rank-100 matrices, equivalent\nto learning an embedding of dimension 100 for\neach word. The activation function used for ~hin\nEq. (8) is a hyperbolic tangent function. The computation from the hidden state in the decoder to\nthe output is implemented as a deep neural network (Pascanu et al., 2014) with a single intermediate layer having 500 maxout units each pooling\n2 inputs (Goodfellow et al., 2013).\nAll the weight parameters in the RNN Encoder\u2013\nDecoder were initialized by sampling from an\nisotropic zero-mean (white) Gaussian distribution\nwith its standard deviation \ufb01xed to 0:01, except\nfor the recurrent weight parameters. For the recurrent weight matrices, we \ufb01rst sampled from a\nwhite Gaussian distribution and used its left singular vectors matrix, following (Saxe et al., 2014).\nWe used Adadelta and stochastic gradient\ndescent to train the RNN Encoder\u2013Decoder\nwith hyperparameters \u000f= 10\u00006and\u001a=\n0:95(Zeiler, 2012). At each update, we used 64"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ad-6db4-11ee-b834-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-21", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "with hyperparameters \u000f= 10\u00006and\u001a=\n0:95(Zeiler, 2012). At each update, we used 64\nrandomly selected phrase pairs from a phrase table (which was created from 348M words). The\nmodel was trained for approximately three days.\nDetails of the architecture used in the experiments are explained in more depth in the supplementary material.\n4.1.2 Neural Language Model\nIn order to assess the effectiveness of scoring\nphrase pairs with the proposed RNN Encoder\u2013\nDecoder, we also tried a more traditional approach\nof using a neural network for learning a target\nlanguage model (CSLM) (Schwenk, 2007). Especially, the comparison between the SMT system\nusing CSLM and that using the proposed approach\nof phrase scoring by RNN Encoder\u2013Decoder will\nclarify whether the contributions from multiple\nneural networks in different parts of the SMT sys-tem add up or are redundant.\nWe trained the CSLM model on 7-grams\nfrom the target corpus. Each input word\nwas projected into the embedding space R512,\nand they were concatenated to form a 3072dimensional vector. The concatenated vector was\nfed through two recti\ufb01ed layers (of size 1536 and\n1024) (Glorot et al., 2011). The output layer was"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ae-6db4-11ee-b856-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-22", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "fed through two recti\ufb01ed layers (of size 1536 and\n1024) (Glorot et al., 2011). The output layer was\na simple softmax layer (see Eq. (2)). All the\nweight parameters were initialized uniformly between\u00000:01and0:01, and the model was trained\nuntil the validation perplexity did not improve for\n10 epochs. After training, the language model\nachieved a perplexity of 45.80. The validation set\nwas a random selection of 0.1% of the corpus. The\nmodel was used to score partial translations during the decoding process, which generally leads to\nhigher gains in BLEU score than n-best list rescoring (Vaswani et al., 2013).\nTo address the computational complexity of\nusing a CSLM in the decoder a buffer was\nused to aggregate n-grams during the stacksearch performed by the decoder. Only when\nthe buffer is full, or a stack is about to\nbe pruned, the n-grams are scored by the\nCSLM. This allows us to perform fast matrixmatrix multiplication on GPU using Theano\n(Bergstra et al., 2010; Bastien et al., 2012).\n\u221260 \u221250 \u221240 \u221230 \u221220 \u221210 0\u221214\u221212\u221210\u22128\u22126\u22124\u221220"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18af-6db4-11ee-b461-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-23", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "\u221260 \u221250 \u221240 \u221230 \u221220 \u221210 0\u221214\u221212\u221210\u22128\u22126\u22124\u221220\nRNN Scores (log)TM Scores (log)\nFigure 3: The visualization of phrase pairs according to their scores (log-probabilities) by the RNN\nEncoder\u2013Decoder and the translation model.\n4.2 Quantitative Analysis\nWe tried the following combinations:\n1. Baseline con\ufb01guration\n2. Baseline + RNN\n3. Baseline + CSLM + RNN\n4. Baseline + CSLM + RNN + Word penalty\nSource Translation Model RNN Encoder\u2013Decoder\nat the end of the [a la \ufb01n de la] [ \u00b4r la \ufb01n des ann \u00b4ees] [ \u02c6etre supprim \u00b4es`a la \ufb01n de la][`a la \ufb01n du] [ `a la \ufb01n des] [ `a la \ufb01n de la]\nfor the \ufb01rst time [rc\rpour la premi r\u00a8ere fois] [ \u00b4et\u00b4e donn \u00b4es pour\nla premi `ere fois] [ \u00b4et\u00b4e comm \u00b4emor \u00b4ee pour la"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b0-6db4-11ee-9ed6-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-24", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "la premi `ere fois] [ \u00b4et\u00b4e comm \u00b4emor \u00b4ee pour la\npremi `ere fois][pour la premi `ere fois] [pour la premi `ere fois ,]\n[pour la premi `ere fois que]\nin the United States\nand[?aux?tats-Unis et] [ \u00b4et\u00b4e ouvertes aux \u00b4EtatsUnis et] [ \u00b4et\u00b4e constat \u00b4ees aux \u00b4Etats-Unis et][aux Etats-Unis et] [des Etats-Unis et] [des\n\u00b4Etats-Unis et]\n, as well as [?s , qu\u2019] [ ?s , ainsi que] [ ?re aussi bien que] [, ainsi qu\u2019] [, ainsi que] [, ainsi que les]\none of the most [?t?l\u2019 un des plus] [ ?l\u2019 un des plus] [ \u02c6etre retenue\ncomme un de ses plus][l\u2019 un des] [le] [un des]\n(a) Long, frequent source phrases\nSource Translation Model RNN Encoder\u2013Decoder"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b1-6db4-11ee-95a4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-25", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "(a) Long, frequent source phrases\nSource Translation Model RNN Encoder\u2013Decoder\n, Minister of Communications and Transport[Secr \u00b4etaire aux communications et aux transports :] [Secr \u00b4etaire aux communications et aux\ntransports][Secr \u00b4etaire aux communications et aux transports] [Secr \u00b4etaire aux communications et aux\ntransports :]\ndid not comply with\nthe[vestimentaire , ne correspondaient pas `a des]\n[susmentionn \u00b4ee n\u2019 \u00b4etait pas conforme aux]\n[pr\u00b4esent \u00b4ees n\u2019 \u00b4etaient pas conformes `a la][n\u2019 ont pas respect \u00b4e les] [n\u2019 \u00b4etait pas conforme\naux] [n\u2019 ont pas respect \u00b4e la]\nparts of the world . [c\rgions du monde .] [r \u00b4egions du monde consid\u00b4er\u00b4ees .] [r \u00b4egion du monde consid \u00b4er\u00b4ee .][parties du monde .] [les parties du monde .]\n[des parties du monde .]\nthe past few days . [le petit texte .] [cours des tout derniers jours .]"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b2-6db4-11ee-b6e1-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-26", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "[des parties du monde .]\nthe past few days . [le petit texte .] [cours des tout derniers jours .]\n[les tout derniers jours .][ces derniers jours .] [les derniers jours .] [cours\ndes derniers jours .]\non Friday and Saturday[vendredi et samedi `a la] [vendredi et samedi `a]\n[se d \u00b4eroulera vendredi et samedi ,][le vendredi et le samedi] [le vendredi et samedi]\n[vendredi et samedi]\n(b) Long, rare source phrases\nTable 2: The top scoring target phrases for a small set of source phrases according to the translation\nmodel (direct translation probability) and by the RNN Encoder\u2013Decoder. Source phrases were randomly\nselected from phrases with 4 or more words. ?denotes an incomplete (partial) character. ris a Cyrillic\nletter ghe.\nThe results are presented in Table 1. As expected, adding features computed by neural networks consistently improves the performance over\nthe baseline performance.\nThe best performance was achieved when we\nused both CSLM and the phrase scores from the\nRNN Encoder\u2013Decoder. This suggests that the"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b3-6db4-11ee-bef4-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-27", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "the baseline performance.\nThe best performance was achieved when we\nused both CSLM and the phrase scores from the\nRNN Encoder\u2013Decoder. This suggests that the\ncontributions of the CSLM and the RNN Encoder\u2013\nDecoder are not too correlated and that one can\nexpect better results by improving each method independently. Furthermore, we tried penalizing the\nnumber of words that are unknown to the neural\nnetworks (i.e. words which are not in the shortlist). We do so by simply adding the number of\nunknown words as an additional feature the loglinear model in Eq. (9).3However, in this case we\n3To understand the effect of the penalty, consider the set\nof all words in the 15,000 large shortlist, SL. All words xi=2\nSL are replaced by a special token [UNK]before being scored\nby the neural networks. Hence, the conditional probability of\nanyxi\nt=2SL is actually given by the model as\np(xt= [UNK]jx<t) =p(xt=2SLjx<t)\n=X\nxj\nt=2SLp\u0010\nxj\ntjx<t\u0011\n\u0015p\u0010\nxi\ntjx<t\u0011\n;"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b4-6db4-11ee-a3ce-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-28", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "=X\nxj\nt=2SLp\u0010\nxj\ntjx<t\u0011\n\u0015p\u0010\nxi\ntjx<t\u0011\n;\nwhere x<tis a shorthand notation for xt\u00001; : : : ; x 1.were not able to achieve better performance on the\ntest set, but only on the development set.\n4.3 Qualitative Analysis\nIn order to understand where the performance improvement comes from, we analyze the phrase pair\nscores computed by the RNN Encoder\u2013Decoder\nagainst the corresponding p(fje)from the translation model. Since the existing translation model\nrelies solely on the statistics of the phrase pairs in\nthe corpus, we expect its scores to be better estimated for the frequent phrases but badly estimated\nfor rare phrases. Also, as we mentioned earlier\nin Sec. 3.1, we further expect the RNN Encoder\u2013\nDecoder which was trained without any frequency\ninformation to score the phrase pairs based rather\non the linguistic regularities than on the statistics\nof their occurrences in the corpus.\nWe focus on those pairs whose source phrase is\nlong (more than 3 words per source phrase) and\nAs a result, the probability of words not in the shortlist is\nalways overestimated. It is possible to address this issue by\nbacking off to an existing model that contain non-shortlisted"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b5-6db4-11ee-ba95-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-29", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "always overestimated. It is possible to address this issue by\nbacking off to an existing model that contain non-shortlisted\nwords (see (Schwenk, 2007)) In this paper, however, we opt\nfor introducing a word penalty instead, which counteracts the\nword probability overestimation.\nSource Samples from RNN Encoder\u2013Decoder\nat the end of the [`a la \ufb01n de la] (\u000211)\nfor the \ufb01rst time [pour la premi `ere fois] (\u000224) [pour la premi `ere fois que] (\u00022)\nin the United States and [aux \u00b4Etats-Unis et] (\u00026) [dans les \u00b4Etats-Unis et] (\u00024)\n, as well as [, ainsi que] [,] [ainsi que] [, ainsi qu\u2019] [et UNK]\none of the most [l\u2019 un des plus] (\u00029) [l\u2019 un des] (\u00025) [l\u2019 une des plus] (\u00022)\n(a) Long, frequent source phrases\nSource Samples from RNN Encoder\u2013Decoder\n, Minister of Communications and Transport[ , ministre des communications et le transport] ( \u000213)"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b6-6db4-11ee-8813-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-30", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Source Samples from RNN Encoder\u2013Decoder\n, Minister of Communications and Transport[ , ministre des communications et le transport] ( \u000213)\ndid not comply with the [n\u2019 tait pas conforme aux] [n\u2019 a pas respect l\u2019] ( \u00022) [n\u2019 a pas respect la] ( \u00023)\nparts of the world . [arts du monde .] (\u000211) [des arts du monde .] ( \u00027)\nthe past few days . [quelques jours .] (\u00025) [les derniers jours .] ( \u00025) [ces derniers jours .] ( \u00022)\non Friday and Saturday [vendredi et samedi] ( \u00025) [le vendredi et samedi] ( \u00027) [le vendredi et le samedi] ( \u00024)\n(b) Long, rare source phrases\nTable 3: Samples generated from the RNN Encoder\u2013Decoder for each source phrase used in Table 2. We\nshow the top-5 target phrases out of 50 samples. They are sorted by the RNN Encoder\u2013Decoder scores.\nFigure 4: 2\u2013D embedding of the learned word representation. The left one shows the full embedding"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b7-6db4-11ee-9c7a-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-31", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Figure 4: 2\u2013D embedding of the learned word representation. The left one shows the full embedding\nspace, while the right one shows a zoomed-in view of one region (color\u2013coded). For more plots, see the\nsupplementary material.\nfrequent . For each such source phrase, we look\nat the target phrases that have been scored high\neither by the translation probability p(fje)or\nby the RNN Encoder\u2013Decoder. Similarly, we perform the same procedure with those pairs whose\nsource phrase is long butrare in the corpus.\nTable 2 lists the top- 3target phrases per source\nphrase favored either by the translation model\nor by the RNN Encoder\u2013Decoder. The source\nphrases were randomly chosen among long ones\nhaving more than 4 or 5 words.\nIn most cases, the choices of the target phrases\nby the RNN Encoder\u2013Decoder are closer to actual or literal translations. We can observe that the\nRNN Encoder\u2013Decoder prefers shorter phrases in\ngeneral.\nInterestingly, many phrase pairs were scored\nsimilarly by both the translation model and the\nRNN Encoder\u2013Decoder, but there were as manyother phrase pairs that were scored radically different (see Fig. 3). This could arise from the\nproposed approach of training the RNN Encoder\u2013"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b8-6db4-11ee-91f5-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-32", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "proposed approach of training the RNN Encoder\u2013\nDecoder on a set of unique phrase pairs, discouraging the RNN Encoder\u2013Decoder from learning\nsimply the frequencies of the phrase pairs from the\ncorpus, as explained earlier.\nFurthermore, in Table 3, we show for each of\nthe source phrases in Table 2, the generated samples from the RNN Encoder\u2013Decoder. For each\nsource phrase, we generated 50 samples and show\nthe top-\ufb01ve phrases accordingly to their scores.\nWe can see that the RNN Encoder\u2013Decoder is\nable to propose well-formed target phrases without looking at the actual phrase table. Importantly,\nthe generated phrases do not overlap completely\nwith the target phrases from the phrase table. This\nencourages us to further investigate the possibility\nof replacing the whole or a part of the phrase table\nFigure 5: 2\u2013D embedding of the learned phrase representation. The top left one shows the full representation space (5000 randomly selected points), while the other three \ufb01gures show the zoomed-in view of\nspeci\ufb01c regions (color\u2013coded).\nwith the proposed RNN Encoder\u2013Decoder in the\nfuture.\n4.4 Word and Phrase Representations\nSince the proposed RNN Encoder\u2013Decoder is not"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18b9-6db4-11ee-b452-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-33", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "with the proposed RNN Encoder\u2013Decoder in the\nfuture.\n4.4 Word and Phrase Representations\nSince the proposed RNN Encoder\u2013Decoder is not\nspeci\ufb01cally designed only for the task of machine\ntranslation, here we brie\ufb02y look at the properties\nof the trained model.\nIt has been known for some time that\ncontinuous space language models using\nneural networks are able to learn semantically meaningful embeddings (See, e.g.,\n(Bengio et al., 2003; Mikolov et al., 2013)). Since\nthe proposed RNN Encoder\u2013Decoder also projects\nto and maps back from a sequence of words into\na continuous space vector, we expect to see a\nsimilar property with the proposed model as well.\nThe left plot in Fig. 4 shows the 2\u2013D embedding\nof the words using the word embedding matrix\nlearned by the RNN Encoder\u2013Decoder. The projection was done by the recently proposed BarnesHut-SNE (van der Maaten, 2013). We can clearly\nsee that semantically similar words are clusteredwith each other (see the zoomed-in plots in Fig. 4).\nThe proposed RNN Encoder\u2013Decoder naturally\ngenerates a continuous-space representation of a\nphrase. The representation ( cin Fig. 1) in this"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ba-6db4-11ee-be48-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-34", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "The proposed RNN Encoder\u2013Decoder naturally\ngenerates a continuous-space representation of a\nphrase. The representation ( cin Fig. 1) in this\ncase is a 1000-dimensional vector. Similarly to the\nword representations, we visualize the representations of the phrases that consists of four or more\nwords using the Barnes-Hut-SNE in Fig. 5.\nFrom the visualization, it is clear that the RNN\nEncoder\u2013Decoder captures both semantic and syntactic structures of the phrases. For instance, in\nthe bottom-left plot, most of the phrases are about\nthe duration of time, while those phrases that are\nsyntactically similar are clustered together. The\nbottom-right plot shows the cluster of phrases that\nare semantically similar (countries or regions). On\nthe other hand, the top-right plot shows the phrases\nthat are syntactically similar.\n5 Conclusion\nIn this paper, we proposed a new neural network\narchitecture, called an RNN Encoder\u2013Decoder\nthat is able to learn the mapping from a sequence\nof an arbitrary length to another sequence, possibly from a different set, of an arbitrary length. The\nproposed RNN Encoder\u2013Decoder is able to either\nscore a pair of sequences (in terms of a conditional\nprobability) or generate a target sequence given a\nsource sequence. Along with the new architecture,"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18bb-6db4-11ee-a41e-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-35", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "score a pair of sequences (in terms of a conditional\nprobability) or generate a target sequence given a\nsource sequence. Along with the new architecture,\nwe proposed a novel hidden unit that includes a reset gate and an update gate that adaptively control\nhow much each hidden unit remembers or forgets\nwhile reading/generating a sequence.\nWe evaluated the proposed model with the task\nof statistical machine translation, where we used\nthe RNN Encoder\u2013Decoder to score each phrase\npair in the phrase table. Qualitatively, we were\nable to show that the new model is able to capture linguistic regularities in the phrase pairs well\nand also that the RNN Encoder\u2013Decoder is able to\npropose well-formed target phrases.\nThe scores by the RNN Encoder\u2013Decoder were\nfound to improve the overall translation performance in terms of BLEU scores. Also, we\nfound that the contribution by the RNN Encoder\u2013\nDecoder is rather orthogonal to the existing approach of using neural networks in the SMT system, so that we can improve further the performance by using, for instance, the RNN Encoder\u2013\nDecoder and the neural net language model together.\nOur qualitative analysis of the trained model\nshows that it indeed captures the linguistic regularities in multiple levels i.e. at the word level as\nwell as phrase level. This suggests that there may"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18bc-6db4-11ee-af52-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-36", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Our qualitative analysis of the trained model\nshows that it indeed captures the linguistic regularities in multiple levels i.e. at the word level as\nwell as phrase level. This suggests that there may\nbe more natural language related applications that\nmay bene\ufb01t from the proposed RNN Encoder\u2013\nDecoder.\nThe proposed architecture has large potential\nfor further improvement and analysis. One approach that was not investigated here is to replace the whole, or a part of the phrase table by\nletting the RNN Encoder\u2013Decoder propose target\nphrases. Also, noting that the proposed model is\nnot limited to being used with written language,\nit will be an important future research to apply the\nproposed architecture to other applications such as\nspeech transcription.\nAcknowledgments\nKC, BM, CG, DB and YB would like to thank\nNSERC, Calcul Qu \u00b4ebec, Compute Canada, the\nCanada Research Chairs and CIFAR. FB and HS\nwere partially funded by the European Commis-sion under the project MateCat, and by DARPA\nunder the BOLT project.\nReferences\n[Auli et al.2013] Michael Auli, Michel Galley, Chris\nQuirk, and Geoffrey Zweig. 2013. Joint language\nand translation modeling with recurrent neural networks. In Proceedings of the ACL Conference on\nEmpirical Methods in Natural Language Processing"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18bd-6db4-11ee-93b1-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-37", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Quirk, and Geoffrey Zweig. 2013. Joint language\nand translation modeling with recurrent neural networks. In Proceedings of the ACL Conference on\nEmpirical Methods in Natural Language Processing\n(EMNLP) , pages 1044\u20131054.\n[Axelrod et al.2011] Amittai Axelrod, Xiaodong He,\nand Jianfeng Gao. 2011. Domain adaptation via\npseudo in-domain data selection. In Proceedings of\nthe ACL Conference on Empirical Methods in Natural Language Processing (EMNLP) , pages 355\u2013362.\n[Bastien et al.2012] Fr \u00b4ed\u00b4eric Bastien, Pascal Lamblin,\nRazvan Pascanu, James Bergstra, Ian J. Goodfellow,\nArnaud Bergeron, Nicolas Bouchard, and Yoshua\nBengio. 2012. Theano: new features and speed improvements. Deep Learning and Unsupervised Feature Learning NIPS 2012 Workshop.\n[Bengio et al.2003] Yoshua Bengio, R \u00b4ejean Ducharme,\nPascal Vincent, and Christian Janvin. 2003. A neural probabilistic language model. J. Mach. Learn.\nRes., 3:1137\u20131155, March."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18be-6db4-11ee-afaf-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-38", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Pascal Vincent, and Christian Janvin. 2003. A neural probabilistic language model. J. Mach. Learn.\nRes., 3:1137\u20131155, March.\n[Bengio et al.2013] Y . Bengio, N. BoulangerLewandowski, and R. Pascanu. 2013. Advances\nin optimizing recurrent networks. In Proceedings\nof the 38th International Conference on Acoustics,\nSpeech, and Signal Processing (ICASSP 2013) ,\nMay.\n[Bergstra et al.2010] James Bergstra, Olivier Breuleux,\nFr\u00b4ed\u00b4eric Bastien, Pascal Lamblin, Razvan Pascanu,\nGuillaume Desjardins, Joseph Turian, David WardeFarley, and Yoshua Bengio. 2010. Theano: a CPU\nand GPU math expression compiler. In Proceedings\nof the Python for Scienti\ufb01c Computing Conference\n(SciPy) , June. Oral Presentation.\n[Chandar et al.2014] Sarath Chandar, Stanislas Lauly,\nHugo Larochelle, Mitesh Khapra, Balaraman Ravindran, Vikas Raykar, and Amrita Saha. 2014. An autoencoder approach to learning bilingual word representations. arXiv: 1402.1454 [cs.CL] , February."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18bf-6db4-11ee-9bcb-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-39", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "[Dahl et al.2012] George E. Dahl, Dong Yu, Li Deng,\nand Alex Acero. 2012. Context-dependent pretrained deep neural networks for large vocabulary\nspeech recognition. IEEE Transactions on Audio,\nSpeech, and Language Processing , 20(1):33\u201342.\n[Devlin et al.2014] Jacob Devlin, Rabih Zbib,\nZhongqiang Huang, Thomas Lamar, Richard\nSchwartz, , and John Makhoul. 2014. Fast and\nrobust neural network joint models for statistical\nmachine translation. In Proceedings of the ACL\n2014 Conference , ACL \u201914, pages 1370\u20131380.\n[Gao et al.2013] Jianfeng Gao, Xiaodong He, Wen tau\nYih, and Li Deng. 2013. Learning semantic representations for the phrase translation model. Technical report, Microsoft Research.\n[Glorot et al.2011] X. Glorot, A. Bordes, and Y . Bengio. 2011. Deep sparse recti\ufb01er neural networks. In\nAISTATS\u20192011 .\n[Goodfellow et al.2013] Ian J. Goodfellow, David\nWarde-Farley, Mehdi Mirza, Aaron Courville, and\nYoshua Bengio. 2013. Maxout networks. In\nICML\u20192013 ."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c0-6db4-11ee-840f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-40", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Warde-Farley, Mehdi Mirza, Aaron Courville, and\nYoshua Bengio. 2013. Maxout networks. In\nICML\u20192013 .\n[Graves2012] Alex Graves. 2012. Supervised Sequence Labelling with Recurrent Neural Networks .\nStudies in Computational Intelligence. Springer.\n[Hochreiter and Schmidhuber1997] S. Hochreiter and\nJ. Schmidhuber. 1997. Long short-term memory.\nNeural Computation , 9(8):1735\u20131780.\n[Kalchbrenner and Blunsom2013] Nal Kalchbrenner\nand Phil Blunsom. 2013. Two recurrent continuous\ntranslation models. In Proceedings of the ACL Conference on Empirical Methods in Natural Language\nProcessing (EMNLP) , pages 1700\u20131709.\n[Koehn et al.2003] Philipp Koehn, Franz Josef Och,\nand Daniel Marcu. 2003. Statistical phrase-based\ntranslation. In Proceedings of the 2003 Conference\nof the North American Chapter of the Association\nfor Computational Linguistics on Human Language\nTechnology - Volume 1 , NAACL \u201903, pages 48\u201354.\n[Koehn2005] P. Koehn. 2005. Europarl: A parallel corpus for statistical machine translation. In Machine"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c1-6db4-11ee-b692-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-41", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "[Koehn2005] P. Koehn. 2005. Europarl: A parallel corpus for statistical machine translation. In Machine\nTranslation Summit X , pages 79\u201386, Phuket, Thailand.\n[Krizhevsky et al.2012] Alex Krizhevsky, Ilya\nSutskever, and Geoffrey Hinton. 2012. ImageNet classi\ufb01cation with deep convolutional neural\nnetworks. In Advances in Neural Information\nProcessing Systems 25 (NIPS\u20192012) .\n[Marcu and Wong2002] Daniel Marcu and William\nWong. 2002. A phrase-based, joint probability\nmodel for statistical machine translation. In Proceedings of the ACL-02 Conference on Empirical\nMethods in Natural Language Processing - Volume\n10, EMNLP \u201902, pages 133\u2013139.\n[Mikolov et al.2013] Tomas Mikolov, Ilya Sutskever,\nKai Chen, Greg Corrado, and Jeff Dean. 2013. Distributed representations of words and phrases and\ntheir compositionality. In Advances in Neural Information Processing Systems 26 , pages 3111\u20133119.\n[Moore and Lewis2010] Robert C. Moore and William\nLewis. 2010. Intelligent selection of language\nmodel training data. In Proceedings of the ACL\n2010 Conference Short Papers , ACLShort \u201910,"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c2-6db4-11ee-a613-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-42", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Lewis. 2010. Intelligent selection of language\nmodel training data. In Proceedings of the ACL\n2010 Conference Short Papers , ACLShort \u201910,\npages 220\u2013224, Stroudsburg, PA, USA.[Pascanu et al.2014] R. Pascanu, C. Gulcehre, K. Cho,\nand Y . Bengio. 2014. How to construct deep recurrent neural networks. In Proceedings of the Second\nInternational Conference on Learning Representations (ICLR 2014) , April.\n[Saxe et al.2014] Andrew M. Saxe, James L. McClelland, and Surya Ganguli. 2014. Exact solutions\nto the nonlinear dynamics of learning in deep linear neural networks. In Proceedings of the Second\nInternational Conference on Learning Representations (ICLR 2014) , April.\n[Schwenk et al.2006] Holger Schwenk, Marta R. CostaJuss`a, and Jos \u00b4e A. R. Fonollosa. 2006. Continuous\nspace language models for the iwslt 2006 task. In\nIWSLT , pages 166\u2013173.\n[Schwenk2007] Holger Schwenk. 2007. Continuous\nspace language models. Comput. Speech Lang. ,\n21(3):492\u2013518, July.\n[Schwenk2012] Holger Schwenk. 2012. Continuous"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c3-6db4-11ee-85d7-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-43", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "space language models. Comput. Speech Lang. ,\n21(3):492\u2013518, July.\n[Schwenk2012] Holger Schwenk. 2012. Continuous\nspace translation models for phrase-based statistical machine translation. In Martin Kay and Christian Boitet, editors, Proceedings of the 24th International Conference on Computational Linguistics\n(COLIN) , pages 1071\u20131080.\n[Socher et al.2011] Richard Socher, Eric H. Huang, Jeffrey Pennington, Andrew Y . Ng, and Christopher D.\nManning. 2011. Dynamic pooling and unfolding\nrecursive autoencoders for paraphrase detection. In\nAdvances in Neural Information Processing Systems\n24.\n[Son et al.2012] Le Hai Son, Alexandre Allauzen, and\nFranc \u00b8ois Yvon. 2012. Continuous space translation models with neural networks. In Proceedings of\nthe 2012 Conference of the North American Chapter of the Association for Computational Linguistics:\nHuman Language Technologies , NAACL HLT \u201912,\npages 39\u201348, Stroudsburg, PA, USA.\n[van der Maaten2013] Laurens van der Maaten. 2013.\nBarnes-hut-sne. In Proceedings of the First International Conference on Learning Representations\n(ICLR 2013) , May."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c4-6db4-11ee-9b82-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-44", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Barnes-hut-sne. In Proceedings of the First International Conference on Learning Representations\n(ICLR 2013) , May.\n[Vaswani et al.2013] Ashish Vaswani, Yinggong Zhao,\nVictoria Fossum, and David Chiang. 2013. Decoding with large-scale neural language models improves translation. Proceedings of the Conference\non Empirical Methods in Natural Language Processing , pages 1387\u20131392.\n[Zeiler2012] Matthew D. Zeiler. 2012. ADADELTA:\nan adaptive learning rate method. Technical report,\narXiv 1212.5701.\n[Zou et al.2013] Will Y . Zou, Richard Socher,\nDaniel M. Cer, and Christopher D. Manning.\n2013. Bilingual word embeddings for phrase-based\nmachine translation. In Proceedings of the ACL\nConference on Empirical Methods in Natural\nLanguage Processing (EMNLP) , pages 1393\u20131398.\nA RNN Encoder\u2013Decoder\nIn this document, we describe in detail the architecture of the RNN Encoder\u2013Decoder used in the experiments.\nLet us denote an source phrase by X= (x1;x2;:::;xN)and a target phrase by Y="}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c5-6db4-11ee-b5ec-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-45", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "Let us denote an source phrase by X= (x1;x2;:::;xN)and a target phrase by Y=\n(y1;y2;:::;yM). Each phrase is a sequence of K-dimensional one-hot vectors, such that only one\nelement of the vector is 1and all the others are 0. The index of the active ( 1) element indicates the word\nrepresented by the vector.\nA.1 Encoder\nEach word of the source phrase is embedded in a 500-dimensional vector space: e(xi)2R500.e(x)is\nused in Sec. 4.4 to visualize the words.\nThe hidden state of an encoder consists of 1000 hidden units, and each one of them at time tis\ncomputed by\nhhti\nj=zjhht\u00001i\nj + (1\u0000zj)~hhti\nj;\nwhere\n~hhti\nj= tanh\u0010\n[We(xt)]j+\u0002\nU\u0000\nr\fhht\u00001i\u0001\u0003\nj\u0011\n;\nzj=\u001b\u0010\n[Wze(xt)]j+\u0002\nUzhht\u00001i\u0003\nj\u0011\n;\nrj=\u001b\u0010\n[Wre(xt)]j+\u0002\nUrhht\u00001i\u0003\nj\u0011\n:"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c6-6db4-11ee-a367-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-46", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "j\u0011\n;\nrj=\u001b\u0010\n[Wre(xt)]j+\u0002\nUrhht\u00001i\u0003\nj\u0011\n:\n\u001band\fare a logistic sigmoid function and an element-wise multiplication, respectively. To make the\nequations uncluttered, we omit biases. The initial hidden state hh0i\njis \ufb01xed to 0.\nOnce the hidden state at the Nstep (the end of the source phrase) is computed, the representation of\nthe source phrase cis\nc= tanh\u0010\nVhhNi\u0011\n:\nA.1.1 Decoder\nThe decoder starts by initializing the hidden state with\nh0h0i= tanh\u0000\nV0c\u0001\n;\nwhere we will use\u00010to distinguish parameters of the decoder from those of the encoder.\nThe hidden state at time tof the decoder is computed by\nh0hti\nj=z0jh0ht\u00001i\nj + (1\u0000z0j)~h0hti\nj;\nwhere\n~h0hti\nj= tanh\u0010\u0002\nW0e(yt\u00001)\u0003\nj+r0j\u0002\nU0h0\nht\u00001i+Cc\u0003\u0011\n;\nz0j=\u001b\u0010\u0002"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c7-6db4-11ee-bf9c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-47", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "j+r0j\u0002\nU0h0\nht\u00001i+Cc\u0003\u0011\n;\nz0j=\u001b\u0010\u0002\nW0ze(yt\u00001)\u0003\nj+\u0002\nU0zh0\nht\u00001i\u0003\nj+ [Czc]j\u0011\n;\nr0j=\u001b\u0010\u0002\nW0re(yt\u00001)\u0003\nj+\u0002\nU0rh0\nht\u00001i\u0003\nj+ [Crc]j\u0011\n;\nande(y0)is an all-zero vector. Similarly to the case of the encoder, e(y)is an embedding of a target\nword.\nUnlike the encoder which simply encodes the source phrase, the decoder is learned to generate a target\nphrase. At each time t, the decoder computes the probability of generating j-th word by\np(yt;j= 1jyt\u00001;:::;y1;X) =exp\u0000\ngjshti\u0001\nPK\nj0=1exp\u0000\ngj0shti\u0001;\nwhere thei-element of shtiis\nshti\ni= maxn\ns0hti\n2i\u00001;s0hti\n2io\nand\ns0hti=Ohh0hti+Oyyt\u00001+Occ:"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c8-6db4-11ee-9b81-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.1078-48", "title": "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation", "source": "http://arxiv.org/pdf/1406.1078", "start_index": 0, "chroma:document": "s0hti\n2i\u00001;s0hti\n2io\nand\ns0hti=Ohh0hti+Oyyt\u00001+Occ:\nIn short, the shti\niis a so-called maxout unit.\nFor the computational ef\ufb01ciency, instead of a single-matrix output weight G, we use a product of two\nmatrices such that\nG=GlGr;\nwhere Gl2RK\u0002500andGr2R500\u00021000.\nB Word and Phrase Representations\nHere, we show enlarged plots of the word and phrase representations in Figs. 4\u20135.\nFigure 6: 2\u2013D embedding of the learned word representation. The top left one shows the full embedding space, while the other three \ufb01gures show the zoomed-in\nview of speci\ufb01c regions (color\u2013coded).\nFigure 7: 2\u2013D embedding of the learned phrase representation. The top left one shows the full representation space (1000 randomly selected points), while the\nother three \ufb01gures show the zoomed-in view of speci\ufb01c regions (color\u2013coded)."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18c9-6db4-11ee-b693-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-0", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "1\nSpatial Pyramid Pooling in Deep Convolutional\nNetworks for Visual Recognition\nKaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun\nAbstract \u2014Existing deep convolutional neural networks (CNNs) require a \ufb01xed-size ( e.g., 224\u0002224) input image. This requirement is \u201carti\ufb01cial\u201d and may reduce the recognition accuracy for the images or sub-images of an arbitrary size/scale. In this\nwork, we equip the networks with another pooling strategy, \u201cspatial pyramid pooling\u201d, to eliminate the above requirement. The\nnew network structure, called SPP-net, can generate a \ufb01xed-length representation regardless of image size/scale. Pyramid\npooling is also robust to object deformations. With these advantages, SPP-net should in general improve all CNN-based image\nclassi\ufb01cation methods. On the ImageNet 2012 dataset, we demonstrate that SPP-net boosts the accuracy of a variety of CNN\narchitectures despite their different designs. On the Pascal VOC 2007 and Caltech101 datasets, SPP-net achieves state-of-theart classi\ufb01cation results using a single full-image representation and no \ufb01ne-tuning."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ca-6db4-11ee-9162-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-1", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "The power of SPP-net is also signi\ufb01cant in object detection. Using SPP-net, we compute the feature maps from the entire\nimage only once, and then pool features in arbitrary regions (sub-images) to generate \ufb01xed-length representations for training\nthe detectors. This method avoids repeatedly computing the convolutional features. In processing test images, our method is\n24-102\u0002faster than the R-CNN method, while achieving better or comparable accuracy on Pascal VOC 2007.\nIn ImageNet Large Scale Visual Recognition Challenge (ILSVRC) 2014, our methods rank #2 in object detection and #3 in\nimage classi\ufb01cation among all 38 teams. This manuscript also introduces the improvement made for this competition.\nIndex Terms \u2014Convolutional Neural Networks, Spatial Pyramid Pooling, Image Classi\ufb01cation, Object Detection\nF\n1 I NTRODUCTION\nWe are witnessing a rapid, revolutionary change in\nour vision community, mainly caused by deep convolutional neural networks (CNNs) [1] and the availability of large scale training data [2]. Deep-networksbased approaches have recently been substantially\nimproving upon the state of the art in image classi\ufb01cation [3], [4], [5], [6], object detection [7], [8], [5],"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18cb-6db4-11ee-9d54-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-2", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "improving upon the state of the art in image classi\ufb01cation [3], [4], [5], [6], object detection [7], [8], [5],\nmany other recognition tasks [9], [10], [11], [12], and\neven non-recognition tasks.\nHowever, there is a technical issue in the training\nand testing of the CNNs: the prevalent CNNs require\na\ufb01xed input image size ( e.g., 224\u0002224), which limits\nboth the aspect ratio and the scale of the input image.\nWhen applied to images of arbitrary sizes, current\nmethods mostly \ufb01t the input image to the \ufb01xed size,\neither via cropping [3], [4] or via warping [13], [7],\nas shown in Figure 1 (top). But the cropped region\nmay not contain the entire object, while the warped\ncontent may result in unwanted geometric distortion.\nRecognition accuracy can be compromised due to the\ncontent loss or distortion. Besides, a pre-de\ufb01ned scale\n\u000fK. He and J. Sun are with Microsoft Research, Beijing, China. E-mail:\nfkahe,jiansung@microsoft.com\n\u000fX. Zhang is with Xi\u2019an Jiaotong University, Xi\u2019an, China. Email:"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18cc-6db4-11ee-989f-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-3", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "fkahe,jiansung@microsoft.com\n\u000fX. Zhang is with Xi\u2019an Jiaotong University, Xi\u2019an, China. Email:\nxyz.clx@stu.xjtu.edu.cn\n\u000fS. Ren is with University of Science and Technology of China, Hefei,\nChina. Email: sqren@mail.ustc.edu.cn\nThis work was done when X. Zhang and S. Ren were interns at Microsoft\nResearch.\ncrop warp\nspatial pyramid poolingcrop / warp\nconv layers image fc layers outputimage conv layers fc layers outputFigure 1: Top: cropping or warping to \ufb01t a \ufb01xed\nsize. Middle: a conventional CNN. Bottom: our spatial\npyramid pooling network structure.\nmay not be suitable when object scales vary. Fixing\ninput sizes overlooks the issues involving scales.\nSo why do CNNs require a \ufb01xed input size? A CNN\nmainly consists of two parts: convolutional layers,\nand fully-connected layers that follow. The convolutional layers operate in a sliding-window manner\nand output feature maps which represent the spatial\narrangement of the activations (Figure 2). In fact, convolutional layers do not require a \ufb01xed image size and"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18cd-6db4-11ee-bee2-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-4", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "and output feature maps which represent the spatial\narrangement of the activations (Figure 2). In fact, convolutional layers do not require a \ufb01xed image size and\ncan generate feature maps of any sizes. On the other\nhand, the fully-connected layers need to have \ufb01xedsize/length input by their de\ufb01nition. Hence, the \ufb01xedsize constraint comes only from the fully-connected\nlayers, which exist at a deeper stage of the network.\nIn this paper, we introduce a spatial pyramid pooling (SPP) [14], [15] layer to remove the \ufb01xed-size\nconstraint of the network. Speci\ufb01cally, we add anarXiv:1406.4729v4  [cs.CV]  23 Apr 2015\n2\nSPP layer on top of the last convolutional layer. The\nSPP layer pools the features and generates \ufb01xedlength outputs, which are then fed into the fullyconnected layers (or other classi\ufb01ers). In other words,\nwe perform some information \u201caggregation\u201d at a\ndeeper stage of the network hierarchy (between convolutional layers and fully-connected layers) to avoid\nthe need for cropping or warping at the beginning."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18ce-6db4-11ee-917d-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-5", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "deeper stage of the network hierarchy (between convolutional layers and fully-connected layers) to avoid\nthe need for cropping or warping at the beginning.\nFigure 1 (bottom) shows the change of the network\narchitecture by introducing the SPP layer. We call the\nnew network structure SPP-net .\nSpatial pyramid pooling [14], [15] (popularly\nknown as spatial pyramid matching or SPM [15]), as\nan extension of the Bag-of-Words (BoW) model [16],\nis one of the most successful methods in computer\nvision. It partitions the image into divisions from\n\ufb01ner to coarser levels, and aggregates local features\nin them. SPP has long been a key component in the\nleading and competition-winning systems for classi\ufb01cation ( e.g., [17], [18], [19]) and detection ( e.g., [20])\nbefore the recent prevalence of CNNs. Nevertheless,\nSPP has not been considered in the context of CNNs.\nWe note that SPP has several remarkable properties\nfor deep CNNs: 1) SPP is able to generate a \ufb01xedlength output regardless of the input size, while the\nsliding window pooling used in the previous deep\nnetworks [3] cannot; 2) SPP uses multi-level spatial"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18cf-6db4-11ee-ba28-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-6", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "sliding window pooling used in the previous deep\nnetworks [3] cannot; 2) SPP uses multi-level spatial\nbins, while the sliding window pooling uses only\na single window size. Multi-level pooling has been\nshown to be robust to object deformations [15]; 3) SPP\ncan pool features extracted at variable scales thanks\nto the \ufb02exibility of input scales. Through experiments\nwe show that all these factors elevate the recognition\naccuracy of deep networks.\nSPP-net not only makes it possible to generate representations from arbitrarily sized images/windows\nfor testing, but also allows us to feed images with\nvarying sizes or scales during training. Training with\nvariable-size images increases scale-invariance and\nreduces over-\ufb01tting. We develop a simple multi-size\ntraining method. For a single network to accept\nvariable input sizes, we approximate it by multiple\nnetworks that share all parameters, while each of\nthese networks is trained using a \ufb01xed input size. In\neach epoch we train the network with a given input\nsize, and switch to another input size for the next\nepoch. Experiments show that this multi-size training\nconverges just as the traditional single-size training,\nand leads to better testing accuracy."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18d0-6db4-11ee-b3d3-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-7", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "epoch. Experiments show that this multi-size training\nconverges just as the traditional single-size training,\nand leads to better testing accuracy.\nThe advantages of SPP are orthogonal to the speci\ufb01c\nCNN designs. In a series of controlled experiments on\nthe ImageNet 2012 dataset, we demonstrate that SPP\nimproves four different CNN architectures in existing\npublications [3], [4], [5] (or their modi\ufb01cations), over\nthe no-SPP counterparts. These architectures have\nvarious \ufb01lter numbers/sizes, strides, depths, or other\ndesigns. It is thus reasonable for us to conjecture\nthat SPP should improve more sophisticated (deeperand larger) convolutional architectures. SPP-net also\nshows state-of-the-art classi\ufb01cation results on Caltech101 [21] and Pascal VOC 2007 [22] using only a\nsingle full-image representation and no \ufb01ne-tuning.\nSPP-net also shows great strength in object detection. In the leading object detection method R-CNN\n[7], the features from candidate windows are extracted\nvia deep convolutional networks. This method shows\nremarkable detection accuracy on both the VOC and\nImageNet datasets. But the feature computation in RCNN is time-consuming, because it repeatedly applies"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18d1-6db4-11ee-b04c-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-8", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "remarkable detection accuracy on both the VOC and\nImageNet datasets. But the feature computation in RCNN is time-consuming, because it repeatedly applies\nthe deep convolutional networks to the raw pixels\nof thousands of warped regions per image. In this\npaper, we show that we can run the convolutional\nlayers only once on the entire image (regardless of\nthe number of windows), and then extract features\nby SPP-net on the feature maps. This method yields\na speedup of over one hundred times over R-CNN.\nNote that training/running a detector on the feature\nmaps (rather than image regions) is actually a more\npopular idea [23], [24], [20], [5]. But SPP-net inherits\nthe power of the deep CNN feature maps and also the\n\ufb02exibility of SPP on arbitrary window sizes, which\nleads to outstanding accuracy and ef\ufb01ciency. In our\nexperiment, the SPP-net-based system (built upon the\nR-CNN pipeline) computes features 24-102 \u0002faster\nthan R-CNN, while has better or comparable accuracy.\nWith the recent fast proposal method of EdgeBoxes\n[25], our system takes 0.5 seconds processing an image\n(including all steps ). This makes our method practical\nfor real-world applications."}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18d2-6db4-11ee-beba-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-9", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "[25], our system takes 0.5 seconds processing an image\n(including all steps ). This makes our method practical\nfor real-world applications.\nA preliminary version of this manuscript has been\npublished in ECCV 2014. Based on this work, we\nattended the competition of ILSVRC 2014 [26], and\nranked #2 in object detection and #3 in image classi\ufb01cation (both are provided-data-only tracks) among\nall 38 teams. There are a few modi\ufb01cations made\nfor ILSVRC 2014. We show that the SPP-nets can\nboost various networks that are deeper and larger\n(Sec. 3.1.2-3.1.4) over the no-SPP counterparts. Further, driven by our detection framework, we \ufb01nd\nthat multi-view testing on feature maps with \ufb02exibly\nlocated/sized windows (Sec. 3.1.5) can increase the\nclassi\ufb01cation accuracy. This manuscript also provides\nthe details of these modi\ufb01cations.\nWe have released the code to facilitate future research ( http://research.microsoft.com/en-us/um/people/kahe/ ).\n2 D EEP NETWORKS WITH SPATIAL PYRAMIDPOOLING\n2.1 Convolutional Layers and Feature Maps"}


created_at: 2023-10-18 12:45:45
topic: persistent://default/default/052498be-6b77-431d-a531-3048a38fd9e3
id: 2d1a18d3-6db4-11ee-9b77-00155d3fea2c
encoding: FLOAT32
metadata:
{"uid": "1406.4729-10", "title": "Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition", "source": "http://arxiv.org/pdf/1406.4729", "start_index": 0, "chroma:document": "2 D EEP NETWORKS WITH SPATIAL PYRAMIDPOOLING\n2.1 Convolutional Layers and Feature Maps\nConsider the popular seven-layer architectures [3], [4].\nThe \ufb01rst \ufb01ve layers are convolutional, some of which\nare followed by pooling layers. These pooling layers\ncan also be considered as \u201cconvolutional\u201d, in the sense\nthat they are using sliding windows. The last two\n3\n  \n  \nfilter #175\nfilter #55\n(a) image (b) feature maps (c) strongest activations\n  \n  \nfilter #66\nfilter #118\n(a) image (b) feature maps (c) strongest activations\nFigure 2: Visualization of the feature maps. (a) Two images in Pascal VOC 2007. (b) The feature maps of some\nconv 5\ufb01lters. The arrows indicate the strongest responses and their corresponding positions in the images.\n(c) The ImageNet images that have the strongest responses of the corresponding \ufb01lters. The green rectangles\nmark the receptive \ufb01elds of the strongest responses.\nlayers are fully connected, with an N-way softmax as\nthe output, where N is the number of categories.\nThe deep network described above needs a \ufb01xed"}


