Author: rmannibucau Date: Fri Oct 18 16:23:12 2013 New Revision: 1533537 URL: http://svn.apache.org/r1533537 Log: adding base for a collector module, ATM counter data are not that consistent + tests need to be written
Added: commons/sandbox/monitoring/trunk/collector/ commons/sandbox/monitoring/trunk/collector/pom.xml commons/sandbox/monitoring/trunk/collector/src/ commons/sandbox/monitoring/trunk/collector/src/main/ commons/sandbox/monitoring/trunk/collector/src/main/java/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/Event.java commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/RestCollector.java commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounter.java commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounterStore.java commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorDataStoreFactory.java commons/sandbox/monitoring/trunk/collector/src/main/resources/ commons/sandbox/monitoring/trunk/collector/src/test/ commons/sandbox/monitoring/trunk/collector/src/test/java/ Modified: commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/counters/DefaultCounter.java commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/repositories/DefaultRepository.java commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/store/InMemoryGaugeDataStore.java commons/sandbox/monitoring/trunk/cube/src/main/java/org/apache/commons/monitoring/cube/CubeCounterDataStore.java commons/sandbox/monitoring/trunk/pom.xml Added: commons/sandbox/monitoring/trunk/collector/pom.xml URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/collector/pom.xml?rev=1533537&view=auto ============================================================================== --- commons/sandbox/monitoring/trunk/collector/pom.xml (added) +++ commons/sandbox/monitoring/trunk/collector/pom.xml Fri Oct 18 16:23:12 2013 @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>commons-monitoring-parent</artifactId> + <groupId>org.apache.commons.monitoring</groupId> + <version>1.0-SNAPSHOT</version> + </parent> + + <modelVersion>4.0.0</modelVersion> + + <artifactId>commons-monitoring-collector</artifactId> + <name>Commons Monitoring (Sandbox) :: Collector</name> + + <dependencies> + <dependency> + <groupId>org.apache.geronimo.specs</groupId> + <artifactId>geronimo-jaxrs_1.1_spec</artifactId> + <version>1.0</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.commons.monitoring</groupId> + <artifactId>commons-monitoring-core</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.jaxrs</groupId> + <artifactId>jackson-jaxrs-json-provider</artifactId> + <version>2.2.2</version> + </dependency> + + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + </dependencies> +</project> Added: commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/Event.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/Event.java?rev=1533537&view=auto ============================================================================== --- commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/Event.java (added) +++ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/Event.java Fri Oct 18 16:23:12 2013 @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.monitoring.collector.rest; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.Date; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class Event { + private String type; + private String marker; // ?? how to map it in our store ?? + private Map<String, Object> data; + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'", timezone = "UTC") + private Date time; + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getMarker() { + return marker; + } + + public void setMarker(final String marker) { + this.marker = marker; + } + + public Date getTime() { + return time; + } + + public void setTime(final Date time) { + this.time = time; + } + + public Map<String, Object> getData() { + return data; + } + + public void setData(final Map<String, Object> data) { + this.data = data; + } +} Added: commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/RestCollector.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/RestCollector.java?rev=1533537&view=auto ============================================================================== --- commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/RestCollector.java (added) +++ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/RestCollector.java Fri Oct 18 16:23:12 2013 @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.monitoring.collector.rest; + +import org.apache.commons.monitoring.Role; +import org.apache.commons.monitoring.collector.rest.store.CollectorCounter; +import org.apache.commons.monitoring.collector.rest.store.CollectorCounterStore; +import org.apache.commons.monitoring.configuration.Configuration; +import org.apache.commons.monitoring.counters.Counter; +import org.apache.commons.monitoring.counters.Unit; +import org.apache.commons.monitoring.store.CounterDataStore; +import org.apache.commons.monitoring.store.GaugeDataStore; +import org.apache.commons.monitoring.store.InMemoryGaugeDataStore; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Map; + +// close to cube collector API but backed by sirona DataStore +@Consumes(MediaType.APPLICATION_JSON) +@Path("/event") +public class RestCollector { + private static final String OK = "{}"; + private static final String GAUGE = "gauge"; + private static final String COUNTER = "counter"; + + private final CounterDataStore counterDataStore; + private final GaugeDataStore gaugeDataStore; + + public RestCollector() { + gaugeDataStore = Configuration.findOrCreateInstance(GaugeDataStore.class); + if (!InMemoryGaugeDataStore.class.isInstance(gaugeDataStore)) { + throw new IllegalStateException("Collector only works with " + InMemoryGaugeDataStore.class.getName()); + } + + counterDataStore = Configuration.findOrCreateInstance(CounterDataStore.class); + if (!CollectorCounterStore.class.isInstance(counterDataStore)) { + throw new IllegalStateException("Collector only works with " + CollectorCounterStore.class.getName()); + } + } + + @POST + @Path("put") + public Response put(final Event[] events) { + if (events != null && events.length > 0) { + try { + doPut(events); + } catch (final Exception e) { + return error(e); + } + } + return ok(); + } + + private void doPut(final Event[] events) { + for (final Event event : events) { + if (COUNTER.equals(event.getType())) { + updateCounter(event); + } else if (GAUGE.equals(event.getType())) { + updateGauge(event); + } + } + } + + private void updateGauge(final Event event) { + final Map<String,Object> data = event.getData(); + + final long time = event.getTime().getTime(); + + final String role = String.class.cast(data.get("role")); + final String unit = String.class.cast(data.get("unit")); + final double value= Number.class.cast(data.get("value")).doubleValue(); + + final Role roleInstance = new Role(role, Unit.get(unit)); + gaugeDataStore.createOrNoopGauge(roleInstance); + InMemoryGaugeDataStore.class.cast(gaugeDataStore).addToGauge(roleInstance, time, value); + } + + private void updateCounter(final Event event) { + final Map<String,Object> data = event.getData(); + + final long time = event.getTime().getTime(); + + final long hits = Number.class.cast(data.get("hits")).longValue(); + final long sum = Number.class.cast(data.get("sum")).longValue(); + final int concurrency = Number.class.cast(data.get("concurrency")).intValue(); + + final String role = String.class.cast(data.get("role")); + final String unit = String.class.cast(data.get("unit")); + final String name = String.class.cast(data.get("name")); + + final Counter counter = counterDataStore.getOrCreateCounter(new Counter.Key(new Role(role, Unit.get(unit)), name)); + CollectorCounter.class.cast(counter).addEvent(time, hits, sum, concurrency); // we checked the store in the constructor so that's ok + } + + private Response ok() { + return Response.ok(OK).build(); + } + + private Response error(final Exception e) { + return Response.status(400).entity("{\"error\":\"" + e.getMessage() + "\"}").build(); + } +} Added: commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounter.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounter.java?rev=1533537&view=auto ============================================================================== --- commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounter.java (added) +++ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounter.java Fri Oct 18 16:23:12 2013 @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.monitoring.collector.rest.store; + +import org.apache.commons.monitoring.counters.DefaultCounter; +import org.apache.commons.monitoring.store.CounterDataStore; + +public class CollectorCounter extends DefaultCounter { + public CollectorCounter(final Key key, final CounterDataStore store) { + super(key, store); + } + + public void addEvent(final long time, final long hits, final long sum, final int concurrency) { + if (hits == 0) { + return; + } + + // TODO: find a better solution? to let hits be correct we consider we add N times the average which is + // mathematically wrong + // a best solution would be to push all raw data + // but it has big impact on the measure side that we don't want + final double avg = sum * 1. / hits; + lock.lock(); + try { + for (long i = 0; i < hits; i++) { + addInternal(avg); + } + updateConcurrency(concurrency); + } finally { + lock.unlock(); + } + } +} Added: commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounterStore.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounterStore.java?rev=1533537&view=auto ============================================================================== --- commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounterStore.java (added) +++ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorCounterStore.java Fri Oct 18 16:23:12 2013 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.monitoring.collector.rest.store; + +import org.apache.commons.monitoring.counters.Counter; +import org.apache.commons.monitoring.store.InMemoryCounterDataStore; + +public class CollectorCounterStore extends InMemoryCounterDataStore { + @Override + protected Counter newCounter(final Counter.Key key) { + return new CollectorCounter(key, this); + } +} Added: commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorDataStoreFactory.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorDataStoreFactory.java?rev=1533537&view=auto ============================================================================== --- commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorDataStoreFactory.java (added) +++ commons/sandbox/monitoring/trunk/collector/src/main/java/org/apache/commons/monitoring/collector/rest/store/CollectorDataStoreFactory.java Fri Oct 18 16:23:12 2013 @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.monitoring.collector.rest.store; + +import org.apache.commons.monitoring.store.DelegateDataStoreFactory; +import org.apache.commons.monitoring.store.InMemoryGaugeDataStore; + +public class CollectorDataStoreFactory extends DelegateDataStoreFactory { + public CollectorDataStoreFactory() { + super(new CollectorCounterStore(), new InMemoryGaugeDataStore()); + } +} Modified: commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/counters/DefaultCounter.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/counters/DefaultCounter.java?rev=1533537&r1=1533536&r2=1533537&view=diff ============================================================================== --- commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/counters/DefaultCounter.java (original) +++ commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/counters/DefaultCounter.java Fri Oct 18 16:23:12 2013 @@ -28,8 +28,8 @@ public class DefaultCounter implements C private final Key key; private final CounterDataStore dataStore; private volatile int maxConcurrency = 0; - private SummaryStatistics statistics; - private Lock lock = new ReentrantLock(); + protected SummaryStatistics statistics; + protected Lock lock = new ReentrantLock(); public DefaultCounter(final Key key, final CounterDataStore store) { this.key = key; Modified: commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/repositories/DefaultRepository.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/repositories/DefaultRepository.java?rev=1533537&r1=1533536&r2=1533537&view=diff ============================================================================== --- commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/repositories/DefaultRepository.java (original) +++ commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/repositories/DefaultRepository.java Fri Oct 18 16:23:12 2013 @@ -41,13 +41,15 @@ public class DefaultRepository implement CounterDataStore counter = null; try { counter = Configuration.findOrCreateInstance(CounterDataStore.class); - } catch (MonitoringException e) { + } catch (final MonitoringException e) { + // no-op } GaugeDataStore gauge = null; try { gauge = Configuration.findOrCreateInstance(GaugeDataStore.class); - } catch (MonitoringException e) { + } catch (final MonitoringException e) { + // no-op } if (counter == null) { Modified: commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/store/InMemoryGaugeDataStore.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/store/InMemoryGaugeDataStore.java?rev=1533537&r1=1533536&r2=1533537&view=diff ============================================================================== --- commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/store/InMemoryGaugeDataStore.java (original) +++ commons/sandbox/monitoring/trunk/core/src/main/java/org/apache/commons/monitoring/store/InMemoryGaugeDataStore.java Fri Oct 18 16:23:12 2013 @@ -27,7 +27,7 @@ import java.util.concurrent.ConcurrentHa import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryGaugeDataStore implements GaugeDataStore { - private final Map<Role, Map<Long, Double>> gauges = new ConcurrentHashMap<Role, Map<Long, Double>>(); + protected final Map<Role, Map<Long, Double>> gauges = new ConcurrentHashMap<Role, Map<Long, Double>>(); @Override public Map<Long, Double> getGaugeValues(GaugeValuesRequest gaugeValuesRequest) { @@ -53,9 +53,13 @@ public class InMemoryGaugeDataStore impl gauges.put(role, new FixedSizedMap()); } + public void addToGauge(final Role role, final long time, final double value) { + gauges.get(role).put(time, value); + } + @Override public void addToGauge(final Gauge gauge, final long time, final double value) { - gauges.get(gauge.role()).put(time, value); + addToGauge(gauge.role(), time, value); } // no perf issues here normally since add is called not that often Modified: commons/sandbox/monitoring/trunk/cube/src/main/java/org/apache/commons/monitoring/cube/CubeCounterDataStore.java URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/cube/src/main/java/org/apache/commons/monitoring/cube/CubeCounterDataStore.java?rev=1533537&r1=1533536&r2=1533537&view=diff ============================================================================== --- commons/sandbox/monitoring/trunk/cube/src/main/java/org/apache/commons/monitoring/cube/CubeCounterDataStore.java (original) +++ commons/sandbox/monitoring/trunk/cube/src/main/java/org/apache/commons/monitoring/cube/CubeCounterDataStore.java Fri Oct 18 16:23:12 2013 @@ -58,6 +58,7 @@ public class CubeCounterDataStore extend cube.buildEvent(events, COUNTER_TYPE, ts, new MapBuilder() .add("name", counter.getKey().getName()) .add("role", counter.getKey().getRole().getName()) + .add("unit", counter.getKey().getRole().getUnit()) // other metrics are not handled by CubeCounter and useless since cube re-aggregate // so to reduce overhead we just store it locally .add("concurrency", MetricData.Concurrency.value(counter)) Modified: commons/sandbox/monitoring/trunk/pom.xml URL: http://svn.apache.org/viewvc/commons/sandbox/monitoring/trunk/pom.xml?rev=1533537&r1=1533536&r2=1533537&view=diff ============================================================================== --- commons/sandbox/monitoring/trunk/pom.xml (original) +++ commons/sandbox/monitoring/trunk/pom.xml Fri Oct 18 16:23:12 2013 @@ -57,6 +57,7 @@ <module>reporting</module> <module>graphite</module> <module>cube</module> + <module>collector</module> </modules> <developers>