diff --git a/lists/referral_domains.txt b/lists/referral_domains.txt index 5d82af9..aa56bc1 100644 --- a/lists/referral_domains.txt +++ b/lists/referral_domains.txt @@ -62,7 +62,15 @@ justdoit.cards free.gg +// Coin Miners +freebitco.in +btconline.io +btcpool.io +baymack.com +outbuck.com + // Scams +giveawaybot.pw oprewards.com free-gg.com rbxfree.com diff --git a/pom.xml b/pom.xml index 3d70f53..38c01ef 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ net.dv8tion JDA - 3.7.1_395 + 3.8.0_425 com.jagrosh diff --git a/src/main/java/com/jagrosh/vortex/Listener.java b/src/main/java/com/jagrosh/vortex/Listener.java index 9b487cf..5444ab9 100644 --- a/src/main/java/com/jagrosh/vortex/Listener.java +++ b/src/main/java/com/jagrosh/vortex/Listener.java @@ -15,6 +15,7 @@ */ package com.jagrosh.vortex; +import com.jagrosh.vortex.logging.MessageCache.CachedMessage; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -80,7 +81,7 @@ vortex.getAutoMod().performAutomod(m); // Store and log the edit - Message old = vortex.getMessageCache().putMessage(m); + CachedMessage old = vortex.getMessageCache().putMessage(m); vortex.getBasicLogger().logMessageEdit(m, old); } } @@ -89,7 +90,7 @@ GuildMessageDeleteEvent gevent = (GuildMessageDeleteEvent) event; // Log the deletion - Message old = vortex.getMessageCache().pullMessage(gevent.getGuild(), gevent.getMessageIdLong()); + CachedMessage old = vortex.getMessageCache().pullMessage(gevent.getGuild(), gevent.getMessageIdLong()); vortex.getBasicLogger().logMessageDelete(old); } else if (event instanceof MessageBulkDeleteEvent) @@ -97,7 +98,7 @@ MessageBulkDeleteEvent gevent = (MessageBulkDeleteEvent) event; // Get the messages we had cached - List logged = gevent.getMessageIds().stream() + List logged = gevent.getMessageIds().stream() .map(id -> vortex.getMessageCache().pullMessage(gevent.getGuild(), Long.parseLong(id))) .filter(m -> m!=null) .collect(Collectors.toList()); diff --git a/src/main/java/com/jagrosh/vortex/Vortex.java b/src/main/java/com/jagrosh/vortex/Vortex.java index d3b9ba3..695e219 100644 --- a/src/main/java/com/jagrosh/vortex/Vortex.java +++ b/src/main/java/com/jagrosh/vortex/Vortex.java @@ -42,12 +42,14 @@ import com.jagrosh.vortex.logging.ModLogger; import com.jagrosh.vortex.logging.TextUploader; import com.jagrosh.vortex.utils.FormatUtil; +import java.util.EnumSet; import java.util.concurrent.TimeUnit; import net.dv8tion.jda.bot.sharding.DefaultShardManagerBuilder; import net.dv8tion.jda.bot.sharding.ShardManager; import net.dv8tion.jda.core.entities.ChannelType; import net.dv8tion.jda.core.exceptions.PermissionException; import net.dv8tion.jda.core.utils.SessionControllerAdapter; +import net.dv8tion.jda.core.utils.cache.CacheFlag; import net.dv8tion.jda.webhook.WebhookClient; import net.dv8tion.jda.webhook.WebhookClientBuilder; @@ -81,14 +83,15 @@ * 5 - database username * 6 - database password * 7 - log webhook url - * 8 - category id + * 8 - guild id : category id * 9 - number of shards */ List tokens = Files.readAllLines(Paths.get("config.txt")); waiter = new EventWaiter(Executors.newSingleThreadScheduledExecutor(), false); - threadpool = Executors.newScheduledThreadPool(40); + threadpool = Executors.newScheduledThreadPool(50); database = new Database(tokens.get(4), tokens.get(5), tokens.get(6)); - uploader = new TextUploader(this, Long.parseLong(tokens.get(8))); + String[] split = tokens.get(8).split(":"); + uploader = new TextUploader(this, Long.parseLong(split[0].trim()), Long.parseLong(split[1].trim())); modlog = new ModLogger(this); basiclog = new BasicLogger(this); messages = new MessageCache(); @@ -191,6 +194,7 @@ .setGame(Game.playing("loading...")) .setBulkDeleteSplittingEnabled(false) .setRequestTimeoutRetry(true) + .setDisabledCacheFlags(EnumSet.of(CacheFlag.EMOTE)) .setSessionController(new SessionControllerAdapter() { @Override @@ -200,7 +204,7 @@ { if (workerHandle == null) { - workerHandle = new SessionControllerAdapter.QueueWorker(15); + workerHandle = new SessionControllerAdapter.QueueWorker(20); System.gc(); workerHandle.start(); } @@ -287,7 +291,7 @@ if(!g.isAvailable()) return false; int botcount = (int)g.getMembers().stream().filter(m -> m.getUser().isBot()).count(); - if(g.getMembers().size()-botcount<7 || (botcount>20 && ((double)botcount/g.getMembers().size())>0.65)) + if(g.getMembers().size()-botcount<10 || (botcount>20 && ((double)botcount/g.getMembers().size())>0.65)) { if(database.settings.hasSettings(g)) return false; diff --git a/src/main/java/com/jagrosh/vortex/automod/AutoMod.java b/src/main/java/com/jagrosh/vortex/automod/AutoMod.java index 06d6131..bea690b 100644 --- a/src/main/java/com/jagrosh/vortex/automod/AutoMod.java +++ b/src/main/java/com/jagrosh/vortex/automod/AutoMod.java @@ -19,6 +19,7 @@ import com.jagrosh.vortex.Vortex; import com.jagrosh.vortex.database.managers.AutomodManager; import com.jagrosh.vortex.database.managers.AutomodManager.AutomodSettings; +import com.jagrosh.vortex.logging.MessageCache.CachedMessage; import com.jagrosh.vortex.utils.FixedCache; import com.jagrosh.vortex.utils.OtherUtil; import java.time.OffsetDateTime; @@ -278,7 +279,7 @@ if(offenses==settings.dupeDeleteThresh) { shouldChannelMute = "Please stop spamming."; - purgeMessages(message.getGuild(), m -> m.getAuthor().getIdLong()==message.getAuthor().getIdLong() && m.getCreationTime().plusMinutes(2).isAfter(now)); + purgeMessages(message.getGuild(), m -> m.getAuthorId()==message.getAuthor().getIdLong() && m.getCreationTime().plusMinutes(2).isAfter(now)); } else if(offenses>settings.dupeDeleteThresh) shouldDelete = true; @@ -502,13 +503,15 @@ } } - private void purgeMessages(Guild guild, Predicate predicate) + private void purgeMessages(Guild guild, Predicate predicate) { vortex.getMessageCache().getMessages(guild, predicate).forEach(m -> { try { - m.delete().queue(s->{}, f->{}); + TextChannel mtc = m.getTextChannel(guild); + if(mtc!=null) + mtc.deleteMessageById(m.getIdLong()).queue(s->{}, f->{}); } catch(PermissionException ex) {} }); diff --git a/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java b/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java index 17cbe24..89cc603 100644 --- a/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java +++ b/src/main/java/com/jagrosh/vortex/database/managers/AutomodManager.java @@ -100,7 +100,7 @@ ? "Disabled\n\n" : "Max Lines / Message: "+(settings.maxLines==0 ? "Disabled\n" : "`"+settings.maxLines+"`\n") + "Copypasta: `" + settings.copypastaStrikes + " " + Action.STRIKE.getEmoji() + "`\n" + - "Attempted @\u0435veryone: `" + settings.everyoneStrikes + " " + Action.STRIKE.getEmoji() + "`\n\n") + "@\u0435veryone Attempt: `" + settings.everyoneStrikes + " " + Action.STRIKE.getEmoji() + "`\n\n") + "__Miscellaneous__\n" + "Auto AntiRaid: " + (settings.useAutoRaidMode() ? "`" + settings.raidmodeNumber + "` joins/`" + settings.raidmodeTime + "`s\n" diff --git a/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java b/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java index b4fd837..02e995b 100644 --- a/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java +++ b/src/main/java/com/jagrosh/vortex/logging/BasicLogger.java @@ -17,6 +17,7 @@ import com.jagrosh.vortex.utils.AvatarUtil; import com.jagrosh.vortex.Vortex; +import com.jagrosh.vortex.logging.MessageCache.CachedMessage; import com.jagrosh.vortex.utils.FormatUtil; import com.jagrosh.vortex.utils.LogUtil; import java.awt.Color; @@ -28,10 +29,7 @@ import net.dv8tion.jda.core.EmbedBuilder; import net.dv8tion.jda.core.MessageBuilder; import net.dv8tion.jda.core.Permission; -import net.dv8tion.jda.core.entities.Message; -import net.dv8tion.jda.core.entities.MessageEmbed; -import net.dv8tion.jda.core.entities.PermissionOverride; -import net.dv8tion.jda.core.entities.TextChannel; +import net.dv8tion.jda.core.entities.*; import net.dv8tion.jda.core.events.guild.member.GuildMemberJoinEvent; import net.dv8tion.jda.core.events.guild.member.GuildMemberLeaveEvent; import net.dv8tion.jda.core.events.guild.voice.GuildVoiceJoinEvent; @@ -86,11 +84,12 @@ // Message Logs - public void logMessageEdit(Message newMessage, Message oldMessage) + public void logMessageEdit(Message newMessage, CachedMessage oldMessage) { if(oldMessage==null) return; - PermissionOverride po = oldMessage.getTextChannel().getPermissionOverride(oldMessage.getGuild().getSelfMember()); + TextChannel mtc = oldMessage.getTextChannel(vortex.getShardManager()); + PermissionOverride po = mtc.getPermissionOverride(mtc.getGuild().getSelfMember()); if(po!=null && po.getDenied().contains(Permission.MESSAGE_HISTORY)) return; TextChannel tc = vortex.getDatabase().settings.getSettings(newMessage.getGuild()).getMessageLogChannel(newMessage.getGuild()); @@ -111,14 +110,18 @@ FormatUtil.formatFullUser(newMessage.getAuthor())+" edited a message in "+newMessage.getTextChannel().getAsMention()+":", edit.build()); } - public void logMessageDelete(Message oldMessage) + public void logMessageDelete(CachedMessage oldMessage) { if(oldMessage==null) return; - PermissionOverride po = oldMessage.getTextChannel().getPermissionOverride(oldMessage.getGuild().getSelfMember()); + Guild guild = oldMessage.getGuild(vortex.getShardManager()); + if(guild==null) + return; + TextChannel mtc = oldMessage.getTextChannel(vortex.getShardManager()); + PermissionOverride po = mtc.getPermissionOverride(guild.getSelfMember()); if(po!=null && po.getDenied().contains(Permission.MESSAGE_HISTORY)) return; - TextChannel tc = vortex.getDatabase().settings.getSettings(oldMessage.getGuild()).getMessageLogChannel(oldMessage.getGuild()); + TextChannel tc = vortex.getDatabase().settings.getSettings(guild).getMessageLogChannel(guild); if(tc==null) return; String formatted = FormatUtil.formatMessage(oldMessage); @@ -127,11 +130,12 @@ EmbedBuilder delete = new EmbedBuilder() .setColor(Color.RED) .appendDescription(formatted); - log(OffsetDateTime.now(), tc, "\u274C", - FormatUtil.formatFullUser(oldMessage.getAuthor())+"'s message has been deleted from "+oldMessage.getTextChannel().getAsMention()+":", delete.build()); + User author = oldMessage.getAuthor(vortex.getShardManager()); + String user = author==null ? FormatUtil.formatFullUserId(oldMessage.getAuthorId()) : FormatUtil.formatFullUser(author); + log(OffsetDateTime.now(), tc, "\u274C", user+"'s message has been deleted from "+mtc.getAsMention()+":", delete.build()); } - public void logMessageBulkDelete(List messages, int count, TextChannel text) + public void logMessageBulkDelete(List messages, int count, TextChannel text) { if(count==0) return; @@ -143,7 +147,8 @@ //log(OffsetDateTime.now(), tc, "\uD83D\uDEAE", "**"+count+"** messages were deleted from "+text.getAsMention()+" (**"+messages.size()+"** logged)", null); return; } - PermissionOverride po = messages.get(0).getTextChannel().getPermissionOverride(messages.get(0).getGuild().getSelfMember()); + TextChannel mtc = messages.get(0).getTextChannel(vortex.getShardManager()); + PermissionOverride po = mtc.getPermissionOverride(mtc.getGuild().getSelfMember()); if(po!=null && po.getDenied().contains(Permission.MESSAGE_HISTORY)) return; if(messages.size()==1) @@ -154,11 +159,12 @@ EmbedBuilder delete = new EmbedBuilder() .setColor(Color.RED) .appendDescription(formatted); - log(OffsetDateTime.now(), tc, "\u274C", - FormatUtil.formatFullUser(messages.get(0).getAuthor())+"'s message has been deleted from "+messages.get(0).getTextChannel().getAsMention()+":", delete.build()); + User author = messages.get(0).getAuthor(vortex.getShardManager()); + String user = author==null ? FormatUtil.formatFullUserId(messages.get(0).getAuthorId()) : FormatUtil.formatFullUser(author); + log(OffsetDateTime.now(), tc, "\u274C", user+"'s message has been deleted from "+mtc.getAsMention()+":", delete.build()); return; } - vortex.getTextUploader().upload(LogUtil.logMessagesForwards("Deleted Messages", messages), "DeletedMessages", (view, download) -> + vortex.getTextUploader().upload(LogUtil.logCachedMessagesForwards("Deleted Messages", messages, vortex.getShardManager()), "DeletedMessages", (view, download) -> { log(OffsetDateTime.now(), tc, "\uD83D\uDEAE", "**"+count+"** messages were deleted from "+text.getAsMention()+" (**"+messages.size()+"** logged):", new EmbedBuilder().setColor(Color.RED.darker().darker()) diff --git a/src/main/java/com/jagrosh/vortex/logging/MessageCache.java b/src/main/java/com/jagrosh/vortex/logging/MessageCache.java index c38ac1a..05ec753 100644 --- a/src/main/java/com/jagrosh/vortex/logging/MessageCache.java +++ b/src/main/java/com/jagrosh/vortex/logging/MessageCache.java @@ -21,8 +21,9 @@ import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; -import net.dv8tion.jda.core.entities.Guild; -import net.dv8tion.jda.core.entities.Message; +import net.dv8tion.jda.bot.sharding.ShardManager; +import net.dv8tion.jda.core.entities.*; +import net.dv8tion.jda.core.entities.Message.Attachment; /** * @@ -31,26 +32,91 @@ public class MessageCache { private final static int SIZE = 1000; - private final HashMap> cache = new HashMap<>(); + private final HashMap> cache = new HashMap<>(); - public Message putMessage(Message m) + public CachedMessage putMessage(Message m) { if(!cache.containsKey(m.getGuild().getIdLong())) cache.put(m.getGuild().getIdLong(), new FixedCache<>(SIZE)); - return cache.get(m.getGuild().getIdLong()).put(m.getIdLong(), m); + return cache.get(m.getGuild().getIdLong()).put(m.getIdLong(), new CachedMessage(m)); } - public Message pullMessage(Guild guild, long messageId) + public CachedMessage pullMessage(Guild guild, long messageId) { if(!cache.containsKey(guild.getIdLong())) return null; return cache.get(guild.getIdLong()).pull(messageId); } - public List getMessages(Guild guild, Predicate predicate) + public List getMessages(Guild guild, Predicate predicate) { if(!cache.containsKey(guild.getIdLong())) return Collections.EMPTY_LIST; return cache.get(guild.getIdLong()).getValues().stream().filter(predicate).collect(Collectors.toList()); } + + public class CachedMessage implements ISnowflake + { + private final String content; + private final long id, author, channel, guild; + private final List attachments; + + private CachedMessage(Message message) + { + content = message.getContentRaw(); + id = message.getIdLong(); + author = message.getAuthor().getIdLong(); + channel = message.getChannel().getIdLong(); + guild = message.getGuild()==null ? 0L : message.getGuild().getIdLong(); + attachments = message.getAttachments(); + } + + public String getContentRaw() + { + return content; + } + + public List getAttachments() + { + return attachments; + } + + public User getAuthor(ShardManager shardManager) + { + return shardManager.getUserById(author); + } + + public long getAuthorId() + { + return author; + } + + public TextChannel getTextChannel(ShardManager shardManager) + { + if (guild == 0L) + return null; + Guild g = shardManager.getGuildById(guild); + if (g == null) + return null; + return g.getTextChannelById(channel); + } + + public TextChannel getTextChannel(Guild guild) + { + return guild.getTextChannelById(channel); + } + + public Guild getGuild(ShardManager shardManager) + { + if (guild == 0L) + return null; + return shardManager.getGuildById(guild); + } + + @Override + public long getIdLong() + { + return id; + } + } } diff --git a/src/main/java/com/jagrosh/vortex/logging/TextUploader.java b/src/main/java/com/jagrosh/vortex/logging/TextUploader.java index 627a917..b5f7afb 100644 --- a/src/main/java/com/jagrosh/vortex/logging/TextUploader.java +++ b/src/main/java/com/jagrosh/vortex/logging/TextUploader.java @@ -20,6 +20,7 @@ import java.util.List; import net.dv8tion.jda.core.JDA; import net.dv8tion.jda.core.entities.Category; +import net.dv8tion.jda.core.entities.Guild; import net.dv8tion.jda.core.entities.TextChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,26 +40,22 @@ private final Logger LOG = LoggerFactory.getLogger("Upload"); private final Vortex vortex; private final long categoryId; - private Category category; + private final long guildId; private int index = 0; - public TextUploader(Vortex vortex, long categoryId) + public TextUploader(Vortex vortex, long guildId, long categoryId) { this.vortex = vortex; + this.guildId = guildId; this.categoryId = categoryId; } public void upload(String content, String filename, Result done) { - if(category==null) - { - category = vortex.getShardManager().getCategoryById(categoryId); - if(category==null) - { - LOG.warn("Ignored upload due to unknown category"); - return; - } - } + Guild guild = vortex.getShardManager().getGuildById(guildId); + if(guild==null) + return; + Category category = guild.getCategoryById(categoryId); List list = category.getTextChannels(); list.get(index % list.size()).sendFile(content.getBytes(StandardCharsets.UTF_8), filename+".txt", null).queue( m -> done.consume( diff --git a/src/main/java/com/jagrosh/vortex/utils/BlockingSessionController.java b/src/main/java/com/jagrosh/vortex/utils/BlockingSessionController.java new file mode 100644 index 0000000..12fe711 --- /dev/null +++ b/src/main/java/com/jagrosh/vortex/utils/BlockingSessionController.java @@ -0,0 +1,95 @@ +/* + * Copyright 2018 John Grosh (john.a.grosh@gmail.com). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.jagrosh.vortex.utils; + +import net.dv8tion.jda.core.JDA; +import net.dv8tion.jda.core.utils.JDALogger; +import net.dv8tion.jda.core.utils.SessionControllerAdapter; + +/** + * + * @author John Grosh (john.a.grosh@gmail.com) + */ +public class BlockingSessionController extends SessionControllerAdapter +{ + private final int MAX_DELAY = 20*1000; + + @Override + protected void runWorker() + { + synchronized (lock) + { + if (workerHandle == null) + { + workerHandle = new BlockingQueueWorker(); + workerHandle.start(); + } + } + } + + protected class BlockingQueueWorker extends QueueWorker + { + @Override + public void run() + { + try + { + if (this.delay > 0) + { + final long interval = System.currentTimeMillis() - lastConnect; + if (interval < this.delay) + Thread.sleep(this.delay - interval); + } + } + catch (InterruptedException ex) + { + JDALogger.getLog(SessionControllerAdapter.class).error("Unable to backoff", ex); + } + while (!connectQueue.isEmpty()) + { + SessionConnectNode node = connectQueue.poll(); + try + { + node.run(connectQueue.isEmpty()); + lastConnect = System.currentTimeMillis(); + if (connectQueue.isEmpty()) + break; + if (this.delay > 0) + Thread.sleep(this.delay); + int total = 0; + while(node.getJDA().getStatus() != JDA.Status.CONNECTED + && node.getJDA().getStatus() != JDA.Status.SHUTDOWN + && total < MAX_DELAY) + { + total += 100; + Thread.sleep(100); + } + } + catch (InterruptedException e) + { + JDALogger.getLog(SessionControllerAdapter.class).error("Failed to run node", e); + appendSession(node); + } + } + synchronized (lock) + { + workerHandle = null; + if (!connectQueue.isEmpty()) + runWorker(); + } + } + } +} diff --git a/src/main/java/com/jagrosh/vortex/utils/FormatUtil.java b/src/main/java/com/jagrosh/vortex/utils/FormatUtil.java index 8d65599..06cb94f 100644 --- a/src/main/java/com/jagrosh/vortex/utils/FormatUtil.java +++ b/src/main/java/com/jagrosh/vortex/utils/FormatUtil.java @@ -23,6 +23,7 @@ import net.dv8tion.jda.core.entities.VoiceChannel; import com.jagrosh.vortex.Constants; import com.jagrosh.vortex.Vortex; +import com.jagrosh.vortex.logging.MessageCache.CachedMessage; import java.awt.Color; import java.util.Collections; import java.util.function.Function; @@ -54,6 +55,18 @@ return sb.length()>2048 ? sb.toString().substring(0, 2040) : sb.toString(); } + public static String formatMessage(CachedMessage m) + { + StringBuilder sb = new StringBuilder(m.getContentRaw()); + m.getAttachments().forEach(att -> sb.append("\n").append(att.getUrl())); + return sb.length()>2048 ? sb.toString().substring(0, 2040) : sb.toString(); + } + + public static String formatFullUserId(long userId) + { + return "<@"+userId+"> (ID:"+userId+")"; + } + public static String formatUser(User user) { return filterEveryone("**"+user.getName()+"**#"+user.getDiscriminator()); diff --git a/src/main/java/com/jagrosh/vortex/utils/LogUtil.java b/src/main/java/com/jagrosh/vortex/utils/LogUtil.java index da32389..15be492 100644 --- a/src/main/java/com/jagrosh/vortex/utils/LogUtil.java +++ b/src/main/java/com/jagrosh/vortex/utils/LogUtil.java @@ -16,12 +16,14 @@ package com.jagrosh.vortex.utils; import com.jagrosh.vortex.Action; +import com.jagrosh.vortex.logging.MessageCache.CachedMessage; import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import net.dv8tion.jda.bot.sharding.ShardManager; import net.dv8tion.jda.core.entities.Guild; import net.dv8tion.jda.core.entities.Member; import net.dv8tion.jda.core.entities.Message; @@ -130,6 +132,29 @@ return sb.toString().trim(); } + public static String logCachedMessagesForwards(String title, List messages, ShardManager shardManager) + { + TextChannel deltc = messages.get(0).getTextChannel(shardManager); + Guild delg = deltc.getGuild(); + StringBuilder sb = new StringBuilder("-- "+title+" -- #"+deltc.getName()+" ("+deltc.getId()+") -- "+delg.getName()+" ("+delg.getId()+") --"); + CachedMessage m; + for(int i=0; i sb.append("\n").append(att.getUrl())); + } + return sb.toString().trim(); + } + public static String logMessagesBackwards(String title, List messages) { TextChannel deltc = messages.get(0).getTextChannel();